| get_text_metrics |
Ran ran on a synthesized input |
apple/ml-mkqa/mkqa_eval_util.py code served (permissive licence) · get_code("c095134cad4a24ee") How it ranRan in Syntology's sandbox on this synthesized input (drafted by a language model, Claude Haiku 4.5, prompt python_fixture_v1, through the batch API) and returned dict. The image it ran in was not recorded. If your copy returns the same type on these inputs, it matched what we observed; nothing here checks that. It does NOT mean the output is correct: nothing has compared it to the paper.
import torch
import numpy as np
from collections import Counter, OrderedDict
torch.manual_seed(0)
# Define arguments for get_text_metrics function
predictions = [
"The quick brown fox jumps over the lazy dog",
"Machine learning is a subset of artificial intelligence",
"Natural language processing enables computers to understand text"
]
gold_answers = [
["The quick brown fox jumps over the lazy dog", "a quick brown fox jumps over lazy dog"],
["machine learning is subset of artificial intelligence", "ML is a subset of AI"],
["NLP enables computers to understand text", "natural language processing helps computers understand"]
]
lang = "en"
ARGS = {
"predictions": predictions,
"gold_answers": gold_answers,
"lang": lang,
"serial": True,
"workers": None
}
- Setup block drafted by
- a language model, Claude Haiku 4.5, prompt python_fixture_v1, through the batch API
- Imports
- numpy, torch — import names as written, not package names to install; a name the repository itself provides is not a package
- Returned
- type dict — a dict output has no array shape of its own; none is stated — the instrument coerces any output to an array; for a dict output the recorded ndim 0 and dtype family float describe that coercion, not the output, and are not stated
- Environment
- image not recorded; re-verify against syntology-harvest-sandbox:v5-deps; not recorded for this sample: the harness installs a cuda-to-cpu shim (cuda_shim.py) before every sample and no run records whether a sample needed it; seed: none by the harness; any seed is in the setup block
|
| map_em_value |
Ran ran on a synthesized input |
apple/ml-mkqa/mkqa_eval_util.py code served (permissive licence) · get_code("11dc500c3a9d29b0") How it ranRan in Syntology's sandbox on this synthesized input (drafted by a language model, Claude Haiku 4.5, prompt python_fixture_v1, through the batch API) and returned float, ndim 0, dtype family float, shape () as a later run recorded it. The image it ran in was not recorded. If your copy returns the same type, ndim and shape on these inputs, it matched what we observed; nothing here checks that. It does NOT mean the output is correct: nothing has compared it to the paper.
import re
import string
MIXED_SEGMENTATION_LANGS = ['zh_cn', 'zh_hk', 'zh_tw', 'ja', 'th', 'km']
ARTICLE_REGEX_BY_LANG = {'en': '\\b(a|an|the)\\b', 'es': '\\b(un|una|unos|unas|el|la|los|las)\\b', 'vi': '\\b(của|là|cái|chiếc|những)\\b', 'de': '\\b(ein|eine|einen|einem|eines|einer|der|die|das|den|dem|des)\\b', 'ar': '\\sال^|ال', 'nl': '\\b(de|het|een|des|der|den)\\b', 'sv': '\\b(en|ett)\\b', 'da': '\\b(en|et)\\b', 'no': '\\b(en|et|ei)\\b', 'fr': "\\b(le|la|l'|les|du|de|d'|des|un|une|des)", 'pt': '\\b(o|a|os|as|um|uma|uns|umas)\\b', 'it': "\\b(il|lo|la|l'|i|gli|le|del|dello|della|dell'|dei|degli|degl'|delle|un'|uno|una|un)", 'fi': '\\b(se|yks|yksi)\\b', 'hu': '\\b(a|az|egy)\\b'}
def whitespace_tokenize(text):
return text.split()
def mixed_segmentation(text):
segs_out = []
temp_str = ''
for char in text:
if temp_str != '':
ss = whitespace_tokenize(temp_str)
segs_out.extend(ss)
temp_str = ''
segs_out.append(char)
if temp_str != '':
ss = whitespace_tokenize(temp_str)
segs_out.extend(ss)
return segs_out
def normalize_answer_by_language(s, lang):
"""Lower text, remove punctuation, articles and extra whitespace.
This function is customized by language.
"""
def remove_articles(text, lang):
article_regex = ARTICLE_REGEX_BY_LANG.get(lang)
if article_regex:
return re.sub(article_regex, ' ', text)
else:
return text
def white_space_fix(text, lang):
if lang in MIXED_SEGMENTATION_LANGS:
tokens = mixed_segmentation(text)
else:
tokens = whitespace_tokenize(text)
return ' '.join([t for t in tokens if t.strip() != ''])
def remove_punc(text):
exclude = set(string.punctuation)
return ''.join((ch for ch in text if ch not in exclude))
def lower(text):
return text.lower()
return white_space_fix(remove_articles(remove_punc(lower(s)), lang), lang)
def calculate_em(prediction, gold_answer, language):
norm_pred = normalize_answer_by_language(prediction, language)
norm_answer = normalize_answer_by_language(gold_answer, language)
return int(norm_pred == norm_answer)
def compute_max_score_over_answers(metric_fn, prediction, ground_truths, language):
assert len(ground_truths) > 0, 'Gold truth answers list should never be empty.'
scores_by_answer = [metric_fn(prediction, ground_truth, language) for ground_truth in ground_truths]
return max(scores_by_answer)
def map_em_value(prediction, gold_answers, lang):
em_value = compute_max_score_over_answers(calculate_em, prediction, gold_answers, lang)
return float(em_value)
ARGS = {
'prediction': 'The quick brown fox',
'gold_answers': ['the quick brown fox', 'a quick brown fox'],
'lang': 'en'
}
- Setup block drafted by
- a language model, Claude Haiku 4.5, prompt python_fixture_v1, through the batch API
- Imports
- none outside the standard library — import names as written, not package names to install; a name the repository itself provides is not a package
- Returned
- type float, ndim 0, dtype family float, shape () — shape () as a later run recorded it
- dtype family
- int means an integer dtype; float means any other dtype the instrument saw, bool and complex included (fixture_check.py records only these two families)
- Environment
- image not recorded; re-verify against syntology-harvest-sandbox:v5-deps; not recorded for this sample: the harness installs a cuda-to-cpu shim (cuda_shim.py) before every sample and no run records whether a sample needed it; seed: none by the harness; any seed is in the setup block
|
| map_f1_value |
Ran ran on a synthesized input |
apple/ml-mkqa/mkqa_eval_util.py code served (permissive licence) · get_code("833a3be37f55a115") How it ranRan in Syntology's sandbox on this synthesized input (drafted by a language model, Claude Haiku 4.5, prompt python_fixture_v1, through the batch API) and returned float, ndim 0, dtype family float, shape () as a later run recorded it. The image it ran in was not recorded. If your copy returns the same type, ndim and shape on these inputs, it matched what we observed; nothing here checks that. It does NOT mean the output is correct: nothing has compared it to the paper.
import re
import string
from collections import Counter, OrderedDict
MIXED_SEGMENTATION_LANGS = ['zh_cn', 'zh_hk', 'zh_tw', 'ja', 'th', 'km']
ARTICLE_REGEX_BY_LANG = {'en': '\\b(a|an|the)\\b', 'es': '\\b(un|una|unos|unas|el|la|los|las)\\b', 'vi': '\\b(của|là|cái|chiếc|những)\\b', 'de': '\\b(ein|eine|einen|einem|eines|einer|der|die|das|den|dem|des)\\b', 'ar': '\\sال^|ال', 'nl': '\\b(de|het|een|des|der|den)\\b', 'sv': '\\b(en|ett)\\b', 'da': '\\b(en|et)\\b', 'no': '\\b(en|et|ei)\\b', 'fr': "\\b(le|la|l'|les|du|de|d'|des|un|une|des)", 'pt': '\\b(o|a|os|as|um|uma|uns|umas)\\b', 'it': "\\b(il|lo|la|l'|i|gli|le|del|dello|della|dell'|dei|degli|degl'|delle|un'|uno|una|un)", 'fi': '\\b(se|yks|yksi)\\b', 'hu': '\\b(a|az|egy)\\b'}
def whitespace_tokenize(text):
return text.split()
def mixed_segmentation(text):
segs_out = []
temp_str = ''
for char in text:
if temp_str != '':
ss = whitespace_tokenize(temp_str)
segs_out.extend(ss)
temp_str = ''
segs_out.append(char)
if temp_str != '':
ss = whitespace_tokenize(temp_str)
segs_out.extend(ss)
return segs_out
def normalize_answer_by_language(s, lang):
"""Lower text, remove punctuation, articles and extra whitespace.
This function is customized by language.
"""
def remove_articles(text, lang):
article_regex = ARTICLE_REGEX_BY_LANG.get(lang)
if article_regex:
return re.sub(article_regex, ' ', text)
else:
return text
def white_space_fix(text, lang):
if lang in MIXED_SEGMENTATION_LANGS:
tokens = mixed_segmentation(text)
else:
tokens = whitespace_tokenize(text)
return ' '.join([t for t in tokens if t.strip() != ''])
def remove_punc(text):
exclude = set(string.punctuation)
return ''.join((ch for ch in text if ch not in exclude))
def lower(text):
return text.lower()
return white_space_fix(remove_articles(remove_punc(lower(s)), lang), lang)
def calculate_f1(prediction, gold_answer, language):
gold_toks = normalize_answer_by_language(gold_answer, language).split() if gold_answer else []
pred_toks = normalize_answer_by_language(prediction, language).split() if prediction else []
common = Counter(gold_toks) & Counter(pred_toks)
num_common = sum(common.values())
if len(gold_toks) == 0 or len(pred_toks) == 0:
return int(gold_toks == pred_toks)
if num_common == 0:
return 0.0
recall = 1.0 * num_common / len(gold_toks)
precision = 1.0 * num_common / len(pred_toks)
return 2.0 * precision * recall / (precision + recall)
def compute_max_score_over_answers(metric_fn, prediction, ground_truths, language):
assert len(ground_truths) > 0, 'Gold truth answers list should never be empty.'
scores_by_answer = [metric_fn(prediction, ground_truth, language) for ground_truth in ground_truths]
return max(scores_by_answer)
def map_f1_value(prediction, gold_answers, lang):
f1_value = compute_max_score_over_answers(calculate_f1, prediction, gold_answers, lang)
return float(f1_value)
ARGS = {
'prediction': 'the quick brown fox',
'gold_answers': ['the quick brown fox', 'a quick brown fox'],
'lang': 'en'
}
- Setup block drafted by
- a language model, Claude Haiku 4.5, prompt python_fixture_v1, through the batch API
- Imports
- none outside the standard library — import names as written, not package names to install; a name the repository itself provides is not a package
- Returned
- type float, ndim 0, dtype family float, shape () — shape () as a later run recorded it
- dtype family
- int means an integer dtype; float means any other dtype the instrument saw, bool and complex included (fixture_check.py records only these two families)
- Environment
- image not recorded; re-verify against syntology-harvest-sandbox:v5-deps; not recorded for this sample: the harness installs a cuda-to-cpu shim (cuda_shim.py) before every sample and no run records whether a sample needed it; seed: none by the harness; any seed is in the setup block
|
| read_annotations |
Ran ran on a synthesized input |
apple/ml-mkqa/mkqa_eval.py code served (permissive licence) · get_code("d866f827b0e9464d") How it ranRan in Syntology's sandbox on this synthesized input (drafted by a language model, Claude Haiku 4.5, prompt python_fixture_v1, through the batch API) and returned defaultdict. The image it ran in was not recorded. If your copy returns the same type on these inputs, it matched what we observed; nothing here checks that. It does NOT mean the output is correct: nothing has compared it to the paper.
import collections
import json
import logging
import os
import tempfile
from gzip import GzipFile
# Create a temporary gzipped JSONL file with mock MKQA data
MKQA_LANGUAGES = ['ar', 'da', 'de', 'en', 'es', 'fi', 'fr', 'he', 'hu', 'it', 'ja', 'km', 'ko', 'ms', 'nl', 'no', 'pl', 'pt', 'ru', 'sv', 'th', 'tr', 'vi', 'zh_cn', 'zh_hk', 'zh_tw']
# Create temporary directory and file
temp_dir = tempfile.mkdtemp()
gold_file_path = os.path.join(temp_dir, 'mkqa.jsonl.gz')
# Generate mock MKQA data
with GzipFile(gold_file_path, 'wb') as gz_file:
for example_id in range(10000):
example_data = {
'example_id': example_id,
'answers': {}
}
# Create answers for each language
for lang in MKQA_LANGUAGES:
example_data['answers'][lang] = [
{
'text': f'answer_{example_id}_{lang}_0',
'type': 'entity',
'aliases': [f'alias_{example_id}_{lang}_0']
},
{
'text': f'answer_{example_id}_{lang}_1',
'type': 'date',
'aliases': []
}
]
line = json.dumps(example_data) + '\n'
gz_file.write(line.encode('utf-8'))
ARGS = {
'gold_path': gold_file_path
}
- Setup block drafted by
- a language model, Claude Haiku 4.5, prompt python_fixture_v1, through the batch API
- Imports
- none outside the standard library — import names as written, not package names to install; a name the repository itself provides is not a package
- Returned
- type defaultdict — a defaultdict output has no array shape of its own; none is stated — the instrument coerces any output to an array; for a defaultdict output the recorded ndim 0 and dtype family float describe that coercion, not the output, and are not stated
- Environment
- image not recorded; re-verify against syntology-harvest-sandbox:v5-deps; not recorded for this sample: the harness installs a cuda-to-cpu shim (cuda_shim.py) before every sample and no run records whether a sample needed it; seed: none by the harness; any seed is in the setup block
|
| read_predictions |
Ran ran on a synthesized input |
apple/ml-mkqa/mkqa_eval.py code served (permissive licence) · get_code("e2e26d4764ff96b4") How it ranRan in Syntology's sandbox on this synthesized input (drafted by a language model, Claude Haiku 4.5, prompt python_fixture_v1, through the batch API) and returned dict. The image it ran in was not recorded. If your copy returns the same type on these inputs, it matched what we observed; nothing here checks that. It does NOT mean the output is correct: nothing has compared it to the paper.
import collections
import json
import logging
import os
import tempfile
MKQAPrediction = collections.namedtuple('MKQAPrediction', ['example_id', 'prediction', 'binary_answer', 'no_answer_prob'])
# Create a temporary file with prediction data
temp_dir = tempfile.mkdtemp()
predictions_file = os.path.join(temp_dir, 'predictions.jsonl')
predictions_data = [
{
"example_id": "ex_001",
"prediction": "The answer is yes",
"binary_answer": "yes",
"no_answer_prob": 0.1
},
{
"example_id": "ex_002",
"prediction": "The answer is no",
"binary_answer": "no",
"no_answer_prob": 0.2
},
{
"example_id": "ex_003",
"prediction": "Some prediction text",
"binary_answer": "",
"no_answer_prob": 0.5
}
]
with open(predictions_file, 'w') as f:
for pred in predictions_data:
f.write(json.dumps(pred) + '\n')
ARGS = {
'predictions_path': predictions_file
}
- Setup block drafted by
- a language model, Claude Haiku 4.5, prompt python_fixture_v1, through the batch API
- Imports
- none outside the standard library — import names as written, not package names to install; a name the repository itself provides is not a package
- Returned
- type dict — a dict output has no array shape of its own; none is stated — the instrument coerces any output to an array; for a dict output the recorded ndim 0 and dtype family float describe that coercion, not the output, and are not stated
- Environment
- image not recorded; re-verify against syntology-harvest-sandbox:v5-deps; not recorded for this sample: the harness installs a cuda-to-cpu shim (cuda_shim.py) before every sample and no run records whether a sample needed it; seed: none by the harness; any seed is in the setup block
|