Upload processor

chat_template.json

@@ -1,3 +1,3 @@
 {
-  "chat_template": "{% set image_placeholder = '<|vision_start|><|image_pad|><|vision_end|>' %}\n{% for message in messages %}\n    {#--- Handle User Messages with Template and Examples ---#}\n    {%- if message['role'] == 'user' and template -%}\n        {% if loop.first and message['role'] != 'system' %}\n            {{- '<|im_start|>system\nYou are NuExtract, an information extraction tool created by NuMind.<|im_end|>' }}\n        {% endif %}\n    \n        {{- '<|im_start|>' + message['role'] -}}\n        \n        {#--- Template Section ---#}\n        {{ '\n# Template:' }}\n        {{- '\n' + template + '\n' }}\n        \n        {#--- Examples Section (if provided) ---#}\n        {% if examples -%}\n            {{- '# Examples:' }}\n            {% for example in examples %}\n                {{- '## Input:\n' }}\n                {#--- Handle image examples ---#}\n                {% if example['input'] is mapping and example['input']['type'] == 'image' %}\n                    {{- image_placeholder | trim -}}\n                {% elif example['input'] == '<image>' %}\n                    {{- image_placeholder | trim -}}\n                {% else %}\n                    {{- example['input'] -}}\n                {% endif %}\n                {{- '\n## Output:\n' ~ example['output'] }}\n            {% endfor %}\n        {%- endif %}\n        \n        {#--- Context Section: Handle various content types ---#}\n        {{- '# Context:\n' }}\n        {%- if message['content'] is string -%}\n            {#--- Simple string content ---#}\n            {{- message['content'] | trim -}}\n        {%- elif message['content'] is mapping and message['content']['type'] == 'image' -%}\n            {#--- Single image document ---#}\n            {{- image_placeholder | trim -}}\n        {%- else -%}\n            {#--- List of content items (mixed text/images) ---#}\n            {#--- First, determine what the actual input content is (not ICL images) ---#}\n            {%- set ns = namespace(has_text_input=false, text_content='') -%}\n            \n            {#--- Count content types and identify actual input document ---#}\n            {%- for content in message['content'] -%}\n                {%- if content is mapping and content.get('type') == 'text' -%}\n                    {%- if content.get('text') != '<image>' -%}\n                        {%- set ns.has_text_input = true -%}\n                        {%- set ns.text_content = content['text'] -%}\n                    {%- endif -%}\n                {%- elif content is string -%}\n                    {%- if content != '<image>' -%}\n                        {%- set ns.has_text_input = true -%}\n                        {%- set ns.text_content = content -%}\n                    {%- endif -%}\n                {%- endif -%}\n            {%- endfor -%}\n            \n            {#--- Determine what to output based on actual input type ---#}\n            {%- if ns.has_text_input -%}\n                {#--- Main input is text, so output the text content ---#}\n                {{- ns.text_content | trim -}}\n            {%- else -%}\n                {#--- Main input is image or <image> placeholder ---#}\n                {%- set ns2 = namespace(found_image=false) -%}\n                {%- for content in message['content'] -%}\n                    {%- if content is mapping and content.get('type') == 'image' and not ns2.found_image -%}\n                        {{- image_placeholder | trim -}}\n                        {%- set ns2.found_image = true -%}\n                    {%- elif content is mapping and content.get('type') == 'text' and content.get('text') == '<image>' and not ns2.found_image -%}\n                        {{- image_placeholder | trim -}}\n                        {%- set ns2.found_image = true -%}\n                    {%- elif content is string and content == '<image>' and not ns2.found_image -%}\n                        {{- image_placeholder | trim -}}\n                        {%- set ns2.found_image = true -%}\n                    {%- endif -%}\n                {%- endfor -%}\n            {%- endif -%}\n        {%- endif -%}\n        {{- '<|im_end|>\n'}}\n        \n    {#--- Handle All Other Messages (Assistant, System, etc.) ---#}\n    {% else %}\n        {% if loop.first and message['role'] != 'system' %}\n            {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>' }}\n        {% endif %}\n        \n        {{- '<|im_start|>' + message['role'] + '\n' }}\n        \n        {#--- Same content handling logic as above but without template/examples ---#}\n        {%- if message['content'] is string -%}\n            {{- message['content'] | trim }}\n        {%- elif message['content'] is mapping and message['content']['type'] == 'image' -%}\n            {{- image_placeholder | trim }}\n        {%- else -%}\n            {%- for content in message['content'] -%}\n                {%- if content is string -%}\n                    {{- content | trim -}}\n                {%- elif content is mapping and content.get('type') == 'text' and content.get('text') == '<image>' -%}\n                    {{- image_placeholder | trim }}\n                {%- elif content is mapping and content.get('type') == 'text' -%}\n                    {{- content['text'] | trim -}}\n                {%- elif content is mapping and content.get('type') == 'image' -%}\n                    {# Skip adding image placeholder - it's already in the text #}\n                {%- endif -%}\n            {%- endfor -%}\n        {%- endif -%}\n        {{- '<|im_end|>'}}\n    {% endif %}\n{% endfor -%}\n{#--- Add Generation Prompt if Requested ---#}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant' }}\n{% endif -%}"
+  "chat_template": "{%- set image_placeholder = '<|vision_start|><|image_pad|><|vision_end|>' -%}\n{%- for message in messages -%}\n    {%- if message['role'] == 'user' -%}\n        {%- if loop.first and message['role'] != 'system' -%}\n            {{- '<|im_start|>system\n' -}}\n            {%- if template -%}\n                {#--- If template, extraction task ---#}\n                {{- 'You are NuExtract, an information extraction tool created by NuMind.' -}}\n            {%- else -%}\n                {#--- Else, template generation task ---#}\n                {{- 'You are a helpful assistant.' -}}\n            {%- endif -%}\n            {{ '<|im_end|>\n' }}\n        {%- endif -%}\n        {{- '<|im_start|>' + message['role'] + '\n' -}}\n        {%- if template -%}\n            {#--- Template Section ---#}\n            {{- '# Template:\n' -}}\n            {{- template -}}\n            {{- '\n' -}}\n            \n            {%- if examples -%}\n                {#--- Examples can only exist in the extraction task ---#}\n                {{- '# Examples:\n' -}}\n                {%- for example in examples -%}\n                    {{- '## Input:\n' -}}\n                    {%- if example['input'] is mapping and (example['input']['type'] == 'image' or example['input']['type'] == 'image_url') -%}\n                        {{- image_placeholder | trim -}}\n                    {%- elif example['input'] == '<image>' -%}\n                        {#--- Keep compatibility with <image> for now ---#}\n                        {{- image_placeholder | trim -}}\n                    {%- else -%}\n                        {#--- Text input example ---#}\n                        {{- example['input'] -}}\n                    {%- endif -%}\n                    {{- '\n' -}}\n                    {{- '## Output:\n' -}}\n                    {{- example['output'] -}}\n                    {{- '\n' -}}\n                {%- endfor -%}\n            {%- endif -%}\n            {{- '# Context:\n' -}}\n        {%- endif -%}\n        \n        {%- if message['content'] is string -%}\n            {#--- Simple string content ---#}\n             message['content'] | trim -}}\n        {%- elif message['content'] is mapping and (message['content']['type'] == 'image' or message['content']['type'] == 'image_url') -%}\n            {{- image_placeholder | trim -}}\n        {%- else -%}\n            {#--- List of content items (mixed text/images) ---#}\n            {#--- First, determine what the actual input content is (not ICL images) ---#}\n            {%- set ns = namespace(has_text_input=false, text_content='') -%}\n            \n            {#--- Count content types and identify actual input document ---#}\n            {%- for content in message['content'] -%}\n                {%- if content is mapping and content.get('type') == 'text' -%}\n                    {%- if content.get('text') != '<image>' -%}\n                        {#--- Keep compatibility with <image> for now ---#}\n                        {%- set ns.has_text_input = true -%}\n                        {%- set ns.text_content = content['text'] -%}\n                    {%- endif -%}\n                {%- elif content is string -%}\n                    {%- if content != '<image>' -%}\n                        {#--- Keep compatibility with <image> for now ---#}\n                        {%- set ns.has_text_input = true -%}\n                        {%- set ns.text_content = content -%}\n                    {%- endif -%}\n                {%- endif -%}\n            {%- endfor -%}\n        \n            {#--- Determine what to output based on actual input type ---#}\n            {%- if ns.has_text_input -%}\n                {#--- Main input is text, so output the text content ---#}\n                {{- ns.text_content | trim -}}\n            {%- else -%}\n                {#--- Main input is image or <image> placeholder ---#}\n                {%- set ns2 = namespace(found_image=false) -%}\n                {%- for content in message['content'] -%}\n                    {%- if content is mapping and (content.get('type') == 'image' or content.get('type') == 'image_url') and not ns2.found_image -%}\n                        {{- image_placeholder | trim -}}\n                        {%- set ns2.found_image = true -%}\n                    {%- elif content is mapping and content.get('type') == 'text' and content.get('text') == '<image>' and not ns2.found_image -%}\n                        {#--- Keep compatibility with <image> for now ---#}\n                        {{- image_placeholder | trim -}}\n                        {%- set ns2.found_image = true -%}\n                    {%- elif content is string and content == '<image>' and not ns2.found_image -%}\n                        {#--- Keep compatibility with <image> for now ---#}\n                        {{- image_placeholder | trim -}}\n                        {%- set ns2.found_image = true -%}\n                    {%- endif -%}\n                {%- endfor -%}\n            {%- endif -%}\n        {%- endif -%}\n        {{- '<|im_end|>\n'}}\n    {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n    {{- '<|im_start|>assistant\n' -}}\n{%- endif -%}"
 }

preprocessor_config.json

@@ -14,7 +14,7 @@
     0.26130258,
     0.27577711
   ],
-  "max_pixels": 2352000,
+  "max_pixels": 23000000,
   "merge_size": 2,
   "min_pixels": 200704,
   "patch_size": 14,
@@ -22,8 +22,8 @@
   "resample": 3,
   "rescale_factor": 0.00392156862745098,
   "size": {
-    "longest_edge": 12845056,
-    "shortest_edge": 3136
+    "longest_edge": 23000000,
+    "shortest_edge": 200704
   },
   "temporal_patch_size": 2
 }

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9c5ae00e602b8860cbd784ba82a8aa14e8feecec692e7076590d014d7b7fdafa
-size 11421896
+oid sha256:ba0c439f7be467bf47d12a7e6f9adc6116201056fc60c67f431c679b7c16afc8
+size 11422064

tokenizer_config.json

@@ -1,4 +1,5 @@
 {
+  "_commit_hash": null,
   "add_bos_token": false,
   "add_prefix_space": false,
   "added_tokens_decoder": {
@@ -195,16 +196,19 @@
     "<|video_pad|>"
   ],
   "bos_token": null,
-  "chat_template": "{% set image_placeholder = '<|vision_start|><|image_pad|><|vision_end|>' %}\n{% for message in messages %}\n    {#--- Handle User Messages with Template and Examples ---#}\n    {%- if message['role'] == 'user' and template -%}\n        {% if loop.first and message['role'] != 'system' %}\n            {{- '<|im_start|>system\nYou are NuExtract, an information extraction tool created by NuMind.<|im_end|>' }}\n        {% endif %}\n    \n        {{- '<|im_start|>' + message['role'] -}}\n        \n        {#--- Template Section ---#}\n        {{ '\n# Template:' }}\n        {{- '\n' + template + '\n' }}\n        \n        {#--- Examples Section (if provided) ---#}\n        {% if examples -%}\n            {{- '# Examples:' }}\n            {% for example in examples %}\n                {{- '## Input:\n' }}\n                {#--- Handle image examples ---#}\n                {% if example['input'] is mapping and example['input']['type'] == 'image' %}\n                    {{- image_placeholder | trim -}}\n                {% elif example['input'] == '<image>' %}\n                    {{- image_placeholder | trim -}}\n                {% else %}\n                    {{- example['input'] -}}\n                {% endif %}\n                {{- '\n## Output:\n' ~ example['output'] }}\n            {% endfor %}\n        {%- endif %}\n        \n        {#--- Context Section: Handle various content types ---#}\n        {{- '# Context:\n' }}\n        {%- if message['content'] is string -%}\n            {#--- Simple string content ---#}\n            {{- message['content'] | trim -}}\n        {%- elif message['content'] is mapping and message['content']['type'] == 'image' -%}\n            {#--- Single image document ---#}\n            {{- image_placeholder | trim -}}\n        {%- else -%}\n            {#--- List of content items (mixed text/images) ---#}\n            {#--- First, determine what the actual input content is (not ICL images) ---#}\n            {%- set ns = namespace(has_text_input=false, text_content='') -%}\n            \n            {#--- Count content types and identify actual input document ---#}\n            {%- for content in message['content'] -%}\n                {%- if content is mapping and content.get('type') == 'text' -%}\n                    {%- if content.get('text') != '<image>' -%}\n                        {%- set ns.has_text_input = true -%}\n                        {%- set ns.text_content = content['text'] -%}\n                    {%- endif -%}\n                {%- elif content is string -%}\n                    {%- if content != '<image>' -%}\n                        {%- set ns.has_text_input = true -%}\n                        {%- set ns.text_content = content -%}\n                    {%- endif -%}\n                {%- endif -%}\n            {%- endfor -%}\n            \n            {#--- Determine what to output based on actual input type ---#}\n            {%- if ns.has_text_input -%}\n                {#--- Main input is text, so output the text content ---#}\n                {{- ns.text_content | trim -}}\n            {%- else -%}\n                {#--- Main input is image or <image> placeholder ---#}\n                {%- set ns2 = namespace(found_image=false) -%}\n                {%- for content in message['content'] -%}\n                    {%- if content is mapping and content.get('type') == 'image' and not ns2.found_image -%}\n                        {{- image_placeholder | trim -}}\n                        {%- set ns2.found_image = true -%}\n                    {%- elif content is mapping and content.get('type') == 'text' and content.get('text') == '<image>' and not ns2.found_image -%}\n                        {{- image_placeholder | trim -}}\n                        {%- set ns2.found_image = true -%}\n                    {%- elif content is string and content == '<image>' and not ns2.found_image -%}\n                        {{- image_placeholder | trim -}}\n                        {%- set ns2.found_image = true -%}\n                    {%- endif -%}\n                {%- endfor -%}\n            {%- endif -%}\n        {%- endif -%}\n        {{- '<|im_end|>\n'}}\n        \n    {#--- Handle All Other Messages (Assistant, System, etc.) ---#}\n    {% else %}\n        {% if loop.first and message['role'] != 'system' %}\n            {{- '<|im_start|>system\nYou are a helpful assistant.<|im_end|>' }}\n        {% endif %}\n        \n        {{- '<|im_start|>' + message['role'] + '\n' }}\n        \n        {#--- Same content handling logic as above but without template/examples ---#}\n        {%- if message['content'] is string -%}\n            {{- message['content'] | trim }}\n        {%- elif message['content'] is mapping and message['content']['type'] == 'image' -%}\n            {{- image_placeholder | trim }}\n        {%- else -%}\n            {%- for content in message['content'] -%}\n                {%- if content is string -%}\n                    {{- content | trim -}}\n                {%- elif content is mapping and content.get('type') == 'text' and content.get('text') == '<image>' -%}\n                    {{- image_placeholder | trim }}\n                {%- elif content is mapping and content.get('type') == 'text' -%}\n                    {{- content['text'] | trim -}}\n                {%- elif content is mapping and content.get('type') == 'image' -%}\n                    {# Skip adding image placeholder - it's already in the text #}\n                {%- endif -%}\n            {%- endfor -%}\n        {%- endif -%}\n        {{- '<|im_end|>'}}\n    {% endif %}\n{% endfor -%}\n{#--- Add Generation Prompt if Requested ---#}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant' }}\n{% endif -%}",
+  "chat_template": "{%- set image_placeholder = '<|vision_start|><|image_pad|><|vision_end|>' -%}\n{%- for message in messages -%}\n    {%- if message['role'] == 'user' -%}\n        {%- if loop.first and message['role'] != 'system' -%}\n            {{- '<|im_start|>system\n' -}}\n            {%- if template -%}\n                {#--- If template, extraction task ---#}\n                {{- 'You are NuExtract, an information extraction tool created by NuMind.' -}}\n            {%- else -%}\n                {#--- Else, template generation task ---#}\n                {{- 'You are a helpful assistant.' -}}\n            {%- endif -%}\n            {{ '<|im_end|>\n' }}\n        {%- endif -%}\n        {{- '<|im_start|>' + message['role'] + '\n' -}}\n        {%- if template -%}\n            {#--- Template Section ---#}\n            {{- '# Template:\n' -}}\n            {{- template -}}\n            {{- '\n' -}}\n            \n            {%- if examples -%}\n                {#--- Examples can only exist in the extraction task ---#}\n                {{- '# Examples:\n' -}}\n                {%- for example in examples -%}\n                    {{- '## Input:\n' -}}\n                    {%- if example['input'] is mapping and (example['input']['type'] == 'image' or example['input']['type'] == 'image_url') -%}\n                        {{- image_placeholder | trim -}}\n                    {%- elif example['input'] == '<image>' -%}\n                        {#--- Keep compatibility with <image> for now ---#}\n                        {{- image_placeholder | trim -}}\n                    {%- else -%}\n                        {#--- Text input example ---#}\n                        {{- example['input'] -}}\n                    {%- endif -%}\n                    {{- '\n' -}}\n                    {{- '## Output:\n' -}}\n                    {{- example['output'] -}}\n                    {{- '\n' -}}\n                {%- endfor -%}\n            {%- endif -%}\n            {{- '# Context:\n' -}}\n        {%- endif -%}\n        \n        {%- if message['content'] is string -%}\n            {#--- Simple string content ---#}\n             message['content'] | trim -}}\n        {%- elif message['content'] is mapping and (message['content']['type'] == 'image' or message['content']['type'] == 'image_url') -%}\n            {{- image_placeholder | trim -}}\n        {%- else -%}\n            {#--- List of content items (mixed text/images) ---#}\n            {#--- First, determine what the actual input content is (not ICL images) ---#}\n            {%- set ns = namespace(has_text_input=false, text_content='') -%}\n            \n            {#--- Count content types and identify actual input document ---#}\n            {%- for content in message['content'] -%}\n                {%- if content is mapping and content.get('type') == 'text' -%}\n                    {%- if content.get('text') != '<image>' -%}\n                        {#--- Keep compatibility with <image> for now ---#}\n                        {%- set ns.has_text_input = true -%}\n                        {%- set ns.text_content = content['text'] -%}\n                    {%- endif -%}\n                {%- elif content is string -%}\n                    {%- if content != '<image>' -%}\n                        {#--- Keep compatibility with <image> for now ---#}\n                        {%- set ns.has_text_input = true -%}\n                        {%- set ns.text_content = content -%}\n                    {%- endif -%}\n                {%- endif -%}\n            {%- endfor -%}\n        \n            {#--- Determine what to output based on actual input type ---#}\n            {%- if ns.has_text_input -%}\n                {#--- Main input is text, so output the text content ---#}\n                {{- ns.text_content | trim -}}\n            {%- else -%}\n                {#--- Main input is image or <image> placeholder ---#}\n                {%- set ns2 = namespace(found_image=false) -%}\n                {%- for content in message['content'] -%}\n                    {%- if content is mapping and (content.get('type') == 'image' or content.get('type') == 'image_url') and not ns2.found_image -%}\n                        {{- image_placeholder | trim -}}\n                        {%- set ns2.found_image = true -%}\n                    {%- elif content is mapping and content.get('type') == 'text' and content.get('text') == '<image>' and not ns2.found_image -%}\n                        {#--- Keep compatibility with <image> for now ---#}\n                        {{- image_placeholder | trim -}}\n                        {%- set ns2.found_image = true -%}\n                    {%- elif content is string and content == '<image>' and not ns2.found_image -%}\n                        {#--- Keep compatibility with <image> for now ---#}\n                        {{- image_placeholder | trim -}}\n                        {%- set ns2.found_image = true -%}\n                    {%- endif -%}\n                {%- endfor -%}\n            {%- endif -%}\n        {%- endif -%}\n        {{- '<|im_end|>\n'}}\n    {%- endif -%}\n{%- endfor -%}\n{%- if add_generation_prompt -%}\n    {{- '<|im_start|>assistant\n' -}}\n{%- endif -%}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",
   "extra_special_tokens": {},
-  "max_pixels": 2352000,
+  "max_length": null,
+  "max_pixels": 23000000,
   "min_pixels": 200704,
   "model_max_length": 131072,
+  "pad_to_multiple_of": null,
   "pad_token": "<|endoftext|>",
-  "padding_side": "left",
+  "pad_token_type_id": 0,
+  "padding_side": "right",
   "processor_class": "Qwen2_5_VLProcessor",
   "split_special_tokens": false,
   "tokenizer_class": "Qwen2Tokenizer",