Update according to latest comments on the main PR

config.json

@@ -1,11 +1,80 @@
 {
   "architectures": [
-    "FLAVAForPretraining"
+    "FlavaForPreTraining"
   ],
   "ce_ignore_index": -100,
   "global_backprop_contrastive": true,
   "global_contrastive_weight": 1.0,
   "hidden_size": 768,
+  "image_codebook_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "bad_words_ids": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "freeze": true,
+    "hidden_size": 256,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "input_channels": 3,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "flava_image_codebook",
+    "no_repeat_ngram_size": 0,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_blocks_per_group": 2,
+    "num_groups": 4,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "transformers_version": "4.19.0.dev0",
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "vocab_size": 8192
+  },
+  "image_codebook_config_dict": {},
   "image_config": {
     "_name_or_path": "",
     "add_cross_attention": false,
@@ -77,12 +146,13 @@
     "top_p": 1.0,
     "torch_dtype": null,
     "torchscript": false,
-    "transformers_version": "4.18.0.dev0",
+    "transformers_version": "4.19.0.dev0",
     "typical_p": 1.0,
     "use_bfloat16": false,
     "vocab_size": 8192
   },
-  "image_config_dict": null,
+  "image_config_dict": {},
+  "init_codebook": true,
   "initializer_factor": 1.0,
   "initializer_range": 0.02,
   "itm_weight": 1.0,
@@ -160,13 +230,15 @@
     "top_p": 1.0,
     "torch_dtype": null,
     "torchscript": false,
-    "transformers_version": "4.18.0.dev0",
+    "transformers_version": "4.19.0.dev0",
     "typical_p": 1.0,
     "use_bfloat16": false,
     "use_cls_token": true
   },
-  "multimodal_config_dict": null,
+  "multimodal_config_dict": {},
   "projection_dim": 768,
+  "return_loss": true,
+  "skip_unmasked_multimodal_encoder": true,
   "text_config": {
     "_name_or_path": "",
     "add_cross_attention": false,
@@ -236,13 +308,13 @@
     "top_p": 1.0,
     "torch_dtype": null,
     "torchscript": false,
-    "transformers_version": "4.18.0.dev0",
+    "transformers_version": "4.19.0.dev0",
     "type_vocab_size": 2,
     "typical_p": 1.0,
     "use_bfloat16": false,
     "vocab_size": 30522
   },
-  "text_config_dict": null,
+  "text_config_dict": {},
   "torch_dtype": "float32",
   "transformers_version": null
 }

preprocessor_config.json

@@ -1,9 +1,26 @@
 {
+  "codebook_crop_size": 112,
+  "codebook_do_center_crop": true,
+  "codebook_do_map_pixels": true,
+  "codebook_do_normalize": true,
+  "codebook_do_resize": true,
+  "codebook_image_mean": [
+    0.0,
+    0.0,
+    0.0
+  ],
+  "codebook_image_std": [
+    1.0,
+    1.0,
+    1.0
+  ],
+  "codebook_resample": 1,
+  "codebook_size": 112,
   "crop_size": 224,
   "do_center_crop": true,
   "do_normalize": true,
   "do_resize": true,
-  "feature_extractor_type": "FLAVAFeatureExtractor",
+  "feature_extractor_type": "FlavaFeatureExtractor",
   "image_mean": [
     0.48145466,
     0.4578275,
@@ -19,7 +36,7 @@
   "mask_group_max_patches": null,
   "mask_group_min_aspect_ratio": 0.3,
   "mask_group_min_patches": 16,
-  "processor_class": "FLAVAProcessor",
+  "processor_class": "FlavaProcessor",
   "resample": 3,
   "size": 224,
   "total_mask_patches": 75

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa8315bfc8aa41a9f50a525a8bd9ab00bb9352aa33af9a4fa883101ca7ea00c1
-size 1215620615
+oid sha256:de8cb370b6412f4b3c5a7d89d563e357004b252225dadea42abe54d484c036e6
+size 1430791411

tokenizer_config.json

@@ -1 +1 @@
-{"do_lower_case": true, "do_basic_tokenize": true, "never_split": null, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "bert-base-uncased", "tokenizer_class": "BertTokenizer", "processor_class": "FLAVAProcessor"}
+{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "model_max_length": 512, "special_tokens_map_file": null, "name_or_path": "bert-base-uncased", "tokenizer_class": "BertTokenizer", "processor_class": "FlavaProcessor"}