mirror of
https://github.com/hanzoai/benchmarks.git
synced 2026-10-30 14:59:11 +00:00
zeekay
c9126360b5
Kai's keys hold (one joint family's projected exact moves 0.02 to 0.025); its timings fall. questions.json is bench speed on ra: Kai, Kai with a cold option cache and Laya interleaved call by call, Metal F32 and CPU (Kai's projections Q8_0 there), and Jev's one network run. Claude-Session: https://claude.ai/code/session_018Gt5TvrE2rxXuZikWSauv3
1 line
24 KiB
JSON
1 line
24 KiB
JSON
{"meta":{"run 2026-09-27 09:31:39 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[16.88232421875,25.06396484375,26.1884765625],"at":"2026-09-27 09:31:39 +0000","laya":{"runtime":"bench (hanzo-ml, kai::joint)","checkpoints":[{"id":"laya","repo":"hanzoai/kai-1","revision":"b50502c28537df49a3621f6fa543f9e8521e8a9c"},{"id":"laya-multilingual","repo":"hanzoai/kai-1-multilingual","revision":"3119843b5261e34903f93d03813467524c57982e"},{"id":"laya-agent","repo":"hanzoai/kai-1-agent","revision":"e3e416499b8a983d2aa0bcc684d5457ad6bfd6a1"}],"device":"metal","dtype":"f32"},"jev_usd":0.0,"mimii_members_sha256":"cec140270a8eb25b7e9d6fe253923efd9e817f62f3286f2f516e2fe287166992","sources":{"har":{"url":"https://archive.ics.uci.edu/static/public/240/human+activity+recognition+using+smartphones.zip","digest":"c00b803081a5c797cd5e4b83700a9810b38d53d9d84e01917e090e1fdbc81031","licence":"CC BY 4.0"},"cmapss":{"url":"https://phm-datasets.s3.amazonaws.com/NASA/6.+Turbofan+Engine+Degradation+Simulation+Data+Set.zip","digest":"c9c5dec12a945a82e8bb4446589d7fb3cc057b5e5d81fa1a12e25ee9912ad3b2","licence":"U.S. Government work"},"mimii":{"url":"https://zenodo.org/records/3384388/files/6_dB_valve.zip?download=1","digest":"md5:fe5fb7c337cd701b1d31dc641e621892","licence":"CC BY-SA 4.0"}},"cases":{"har":300,"cmapss":100,"mimii":100},"chars":{"har":2222,"cmapss":2297,"mimii":1649}},"run 2026-09-27 09:42:32 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[32.2861328125,30.11474609375,28.724609375],"at":"2026-09-27 09:42:32 +0000","jev":{"model":"typesafe/jev-1.13","url":"https://openrouter.ai/api/alpha/decisions","price_per_input_token":4.2e-08},"jev_usd":0.039071,"mimii_members_sha256":"cec140270a8eb25b7e9d6fe253923efd9e817f62f3286f2f516e2fe287166992","sources":{"har":{"url":"https://archive.ics.uci.edu/static/public/240/human+activity+recognition+using+smartphones.zip","digest":"c00b803081a5c797cd5e4b83700a9810b38d53d9d84e01917e090e1fdbc81031","licence":"CC BY 4.0"},"cmapss":{"url":"https://phm-datasets.s3.amazonaws.com/NASA/6.+Turbofan+Engine+Degradation+Simulation+Data+Set.zip","digest":"c9c5dec12a945a82e8bb4446589d7fb3cc057b5e5d81fa1a12e25ee9912ad3b2","licence":"U.S. Government work"},"mimii":{"url":"https://zenodo.org/records/3384388/files/6_dB_valve.zip?download=1","digest":"md5:fe5fb7c337cd701b1d31dc641e621892","licence":"CC BY-SA 4.0"}},"cases":{"har":300,"cmapss":100,"mimii":100},"chars":{"har":2222,"cmapss":2297,"mimii":1649}},"run 2026-09-27 09:51:11 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[12.2392578125,20.0908203125,24.2119140625],"at":"2026-09-27 09:51:11 +0000","jev_usd":0.0,"mimii_members_sha256":"cec140270a8eb25b7e9d6fe253923efd9e817f62f3286f2f516e2fe287166992","sources":{"har":{"url":"https://archive.ics.uci.edu/static/public/240/human+activity+recognition+using+smartphones.zip","digest":"c00b803081a5c797cd5e4b83700a9810b38d53d9d84e01917e090e1fdbc81031","licence":"CC BY 4.0"},"cmapss":{"url":"https://phm-datasets.s3.amazonaws.com/NASA/6.+Turbofan+Engine+Degradation+Simulation+Data+Set.zip","digest":"c9c5dec12a945a82e8bb4446589d7fb3cc057b5e5d81fa1a12e25ee9912ad3b2","licence":"U.S. Government work"},"mimii":{"url":"https://zenodo.org/records/3384388/files/6_dB_valve.zip?download=1","digest":"md5:fe5fb7c337cd701b1d31dc641e621892","licence":"CC BY-SA 4.0"}},"cases":{"har":300,"cmapss":100,"mimii":100},"chars":{"har":2222,"cmapss":2297,"mimii":1649}},"run 2026-09-27 14:42:16 +0000":{"host":"arm64-mac","machine":"aarch64","runtime":"bench (hanzo-ml)","load":[6.955078125,8.50439453125,9.96337890625],"at":"2026-09-27 14:42:16 +0000","jev_usd":0.0,"mimii_members_sha256":"cec140270a8eb25b7e9d6fe253923efd9e817f62f3286f2f516e2fe287166992","sources":{"har":{"url":"https://archive.ics.uci.edu/static/public/240/human+activity+recognition+using+smartphones.zip","digest":"c00b803081a5c797cd5e4b83700a9810b38d53d9d84e01917e090e1fdbc81031","licence":"CC BY 4.0"},"cmapss":{"url":"https://phm-datasets.s3.amazonaws.com/NASA/6.+Turbofan+Engine+Degradation+Simulation+Data+Set.zip","digest":"c9c5dec12a945a82e8bb4446589d7fb3cc057b5e5d81fa1a12e25ee9912ad3b2","licence":"U.S. Government work"},"mimii":{"url":"https://zenodo.org/records/3384388/files/6_dB_valve.zip?download=1","digest":"md5:fe5fb7c337cd701b1d31dc641e621892","licence":"CC BY-SA 4.0"}},"cases":{"har":300,"cmapss":100,"mimii":100},"chars":{"har":2222,"cmapss":2297,"mimii":1649}}},"keys":{"cap/media/jev/audio.accuracy":0.84,"cap/media/jev/cmapss.accuracy":0.3,"cap/media/jev/cmapss.brier":0.7616,"cap/media/jev/cmapss.ece":0.1039,"cap/media/jev/cmapss.errors":0,"cap/media/jev/cmapss.macrof1":0.2482,"cap/media/jev/cmapss.p50ms":138.2136,"cap/media/jev/cmapss.unanswered":0,"cap/media/jev/cmapss.usd":0.0081,"cap/media/jev/cmapss.zeroprob":0.0,"cap/media/jev/har.accuracy":0.1533,"cap/media/jev/har.brier":1.0982,"cap/media/jev/har.ece":0.4456,"cap/media/jev/har.errors":0,"cap/media/jev/har.macrof1":0.0509,"cap/media/jev/har.p50ms":138.5756,"cap/media/jev/har.unanswered":0,"cap/media/jev/har.usd":0.0252,"cap/media/jev/har.zeroprob":0.0,"cap/media/jev/image.input":"text only","cap/media/jev/mimii.accuracy":0.84,"cap/media/jev/mimii.auc":0.8756,"cap/media/jev/mimii.brier":0.3455,"cap/media/jev/mimii.ece":0.1971,"cap/media/jev/mimii.errors":0,"cap/media/jev/mimii.macrof1":0.8394,"cap/media/jev/mimii.p50ms":129.9928,"cap/media/jev/mimii.unanswered":0,"cap/media/jev/mimii.usd":0.0057,"cap/media/jev/mimii.zeroprob":0.0,"cap/media/jev/sensor.accuracy":0.19,"cap/media/jev/video.input":"text only","cap/media/laya/audio.accuracy":0.5,"cap/media/laya/cmapss.accuracy":0.17,"cap/media/laya/cmapss.brier":0.7851,"cap/media/laya/cmapss.ece":0.1298,"cap/media/laya/cmapss.macrof1":0.1144,"cap/media/laya/cmapss.msperq":229.74,"cap/media/laya/cmapss.rejected":0.0,"cap/media/laya/cmapss.statecut":1.0,"cap/media/laya/cmapss.unanswered":0,"cap/media/laya/cmapss.zeroprob":0.0,"cap/media/laya/har.accuracy":0.2033,"cap/media/laya/har.brier":0.8797,"cap/media/laya/har.ece":0.1434,"cap/media/laya/har.macrof1":0.0563,"cap/media/laya/har.msperq":250.83,"cap/media/laya/har.rejected":0.0,"cap/media/laya/har.statecut":1.0,"cap/media/laya/har.unanswered":0,"cap/media/laya/har.zeroprob":0.0,"cap/media/laya/image.input":"text only","cap/media/laya/long.cmapss.accuracy":0.14,"cap/media/laya/long.cmapss.brier":0.9598,"cap/media/laya/long.cmapss.ece":0.4087,"cap/media/laya/long.cmapss.macrof1":0.0614,"cap/media/laya/long.cmapss.msperq":320.12,"cap/media/laya/long.cmapss.rejected":0.0,"cap/media/laya/long.cmapss.statecut":0.0,"cap/media/laya/long.cmapss.unanswered":0,"cap/media/laya/long.cmapss.zeroprob":0.0,"cap/media/laya/long.har.accuracy":0.1633,"cap/media/laya/long.har.brier":1.0574,"cap/media/laya/long.har.ece":0.4227,"cap/media/laya/long.har.macrof1":0.0517,"cap/media/laya/long.har.msperq":470.4567,"cap/media/laya/long.har.rejected":0.0,"cap/media/laya/long.har.statecut":0.0,"cap/media/laya/long.har.unanswered":0,"cap/media/laya/long.har.zeroprob":0.0,"cap/media/laya/long.mimii.accuracy":0.5,"cap/media/laya/long.mimii.auc":0.4856,"cap/media/laya/long.mimii.brier":0.8478,"cap/media/laya/long.mimii.ece":0.4171,"cap/media/laya/long.mimii.macrof1":0.3333,"cap/media/laya/long.mimii.msperq":146.97,"cap/media/laya/long.mimii.rejected":0.0,"cap/media/laya/long.mimii.statecut":0.0,"cap/media/laya/long.mimii.unanswered":0,"cap/media/laya/long.mimii.zeroprob":0.0,"cap/media/laya/mimii.accuracy":0.5,"cap/media/laya/mimii.auc":0.5368,"cap/media/laya/mimii.brier":0.5124,"cap/media/laya/mimii.ece":0.1154,"cap/media/laya/mimii.macrof1":0.3333,"cap/media/laya/mimii.msperq":224.64,"cap/media/laya/mimii.rejected":0.0,"cap/media/laya/mimii.statecut":1.0,"cap/media/laya/mimii.unanswered":0,"cap/media/laya/mimii.zeroprob":0.0,"cap/media/laya/sensor.accuracy":0.195,"cap/media/laya/video.input":"text only"},"detail":{"dropped":{"cwru":"no licence or terms of use stated by engineering.case.edu/bearingdatacenter (checked 2026-09-25: welcome, project-history, apparatus-and-procedures, download-data-file, 12k-drive-end-bearing-fault-data, normal-baseline-data); only a copyright notice"},"har":{"laya":{"metrics":{"n":300,"accuracy":0.2033,"macro_f1":0.0563,"ece":0.1434,"brier":0.8797,"nll":1.9498,"mean_confidence":0.3468,"acc_at_50_coverage":0.2067,"unanswered":0,"zero_prob":0.0,"questions":300,"aurc":0.7893,"risk_at_coverage":{"0.1":0.8333,"0.25":0.8,"0.5":0.7933,"0.75":0.7822,"0.9":0.7926,"1.0":0.7967}},"fit":{"questions":300,"rejected":0,"options_cut":0,"state_cut":300},"dropped":0,"route":["english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english"]},"jev":{"metrics":{"n":300,"accuracy":0.1533,"macro_f1":0.0509,"ece":0.4456,"brier":1.0982,"nll":2.8012,"mean_confidence":0.5976,"acc_at_50_coverage":0.2733,"unanswered":0,"zero_prob":0.0,"questions":300,"aurc":0.6793,"risk_at_coverage":{"0.1":0.4222,"0.25":0.6243,"0.5":0.7267,"0.75":0.8,"0.9":0.8333,"1.0":0.8467}},"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":300},"cost_usd":0.025215,"input_tokens":600348}},"cmapss":{"laya":{"metrics":{"n":100,"accuracy":0.17,"macro_f1":0.1144,"ece":0.1298,"brier":0.7851,"nll":1.4632,"mean_confidence":0.2998,"acc_at_50_coverage":0.12,"unanswered":0,"zero_prob":0.0,"questions":100,"aurc":0.884,"risk_at_coverage":{"0.1":0.9,"0.25":0.92,"0.5":0.88,"0.75":0.84,"0.9":0.8333,"1.0":0.83}},"fit":{"questions":100,"rejected":0,"options_cut":0,"state_cut":100},"dropped":0,"route":["english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english"]},"jev":{"metrics":{"n":100,"accuracy":0.3,"macro_f1":0.2482,"ece":0.1039,"brier":0.7616,"nll":1.4143,"mean_confidence":0.3333,"acc_at_50_coverage":0.28,"unanswered":0,"zero_prob":0.0,"questions":100,"aurc":0.7377,"risk_at_coverage":{"0.1":0.8,"0.25":0.8,"0.5":0.7371,"0.75":0.6667,"0.9":0.6762,"1.0":0.7}},"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":100},"cost_usd":0.008139,"input_tokens":193782}},"mimii":{"laya":{"metrics":{"n":100,"accuracy":0.5,"macro_f1":0.3333,"ece":0.1154,"brier":0.5124,"nll":0.7057,"mean_confidence":0.5837,"acc_at_50_coverage":0.48,"unanswered":0,"zero_prob":0.0,"questions":100,"aurc":0.4453,"risk_at_coverage":{"0.1":0.4,"0.25":0.4,"0.5":0.52,"0.75":0.5067,"0.9":0.4889,"1.0":0.5},"auc":0.5368},"fit":{"questions":100,"rejected":0,"options_cut":0,"state_cut":100},"dropped":0,"route":["english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english","english"]},"jev":{"metrics":{"n":100,"accuracy":0.84,"macro_f1":0.8394,"ece":0.1971,"brier":0.3455,"nll":0.5308,"mean_confidence":0.6429,"acc_at_50_coverage":0.86,"unanswered":0,"zero_prob":0.0,"questions":100,"aurc":0.1173,"risk_at_coverage":{"0.1":0.05,"0.25":0.1067,"0.5":0.13,"0.75":0.1367,"0.9":0.1407,"1.0":0.16},"auc":0.8756},"n_errors":0,"error_kinds":{},"errors":[],"served":{"typesafe/jev-1.13-20260917":100},"cost_usd":0.005717,"input_tokens":136109}},"long.har":{"laya":{"metrics":{"n":300,"accuracy":0.1633,"macro_f1":0.0517,"ece":0.4227,"brier":1.0574,"nll":2.3352,"mean_confidence":0.586,"acc_at_50_coverage":0.2133,"unanswered":0,"zero_prob":0.0,"questions":300,"aurc":0.7644,"risk_at_coverage":{"0.1":0.6333,"0.25":0.7067,"0.5":0.7867,"0.75":0.8178,"0.9":0.837,"1.0":0.8367}},"fit":{"questions":300,"rejected":0,"options_cut":0,"state_cut":0},"dropped":0,"route":["multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual"]}},"long.cmapss":{"laya":{"metrics":{"n":100,"accuracy":0.14,"macro_f1":0.0614,"ece":0.4087,"brier":0.9598,"nll":1.9768,"mean_confidence":0.5487,"acc_at_50_coverage":0.1,"unanswered":0,"zero_prob":0.0,"questions":100,"aurc":0.9199,"risk_at_coverage":{"0.1":0.9,"0.25":0.96,"0.5":0.9,"0.75":0.8933,"0.9":0.8778,"1.0":0.86}},"fit":{"questions":100,"rejected":0,"options_cut":0,"state_cut":0},"dropped":0,"route":["multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual"]}},"long.mimii":{"laya":{"metrics":{"n":100,"accuracy":0.5,"macro_f1":0.3333,"ece":0.4171,"brier":0.8478,"nll":1.2948,"mean_confidence":0.9171,"acc_at_50_coverage":0.46,"unanswered":0,"zero_prob":0.0,"questions":100,"aurc":0.5197,"risk_at_coverage":{"0.1":0.6,"0.25":0.52,"0.5":0.54,"0.75":0.5067,"0.9":0.4889,"1.0":0.5},"auc":0.4856},"fit":{"questions":100,"rejected":0,"options_cut":0,"state_cut":0},"dropped":0,"route":["multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual","multilingual"]}}}} |