From a62cf4d2a99b4a7985c61b2a7feb92a82a8218b7 Mon Sep 17 00:00:00 2001 From: Yuren Hao Date: Sat, 1 Aug 2026 14:10:03 -0500 Subject: World Alignment: unpaired cross-modal correspondence by relational identifiability Method: scene states are sets of part states; relation fields are built within each modality and are invariant to how each side labels its own features; the cross-modal bridge is a coupling searched under an energy that is a closed-form functional of one matrix; solving is spectral initialisation followed by exact local refinement. Evidence: in a procedurally generated closed world, blind recovery of a hidden image-caption correspondence reaches 95.3% at 256 scenes against 0.39% chance, and the recovered pairs transfer to 200 held-out scenes at 93.0% exact retrieval with random-pair and shuffled-image controls at or near chance. Cross-modal value correspondence is derived from disjoint corpora rather than declared. On Visual Genome the field correlation reaches 0.656 against the 0.9 that polynomial recovery needs, with the deficit attributed away from segmentation and discretisation. Protocol: no image-text pair enters any objective, optimiser, initialisation, or model selection; hidden pairs score orderings only. Co-Authored-By: Claude --- artifacts/eval_unpaired.json | 322 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 artifacts/eval_unpaired.json (limited to 'artifacts/eval_unpaired.json') diff --git a/artifacts/eval_unpaired.json b/artifacts/eval_unpaired.json new file mode 100644 index 0000000..cdc5ce3 --- /dev/null +++ b/artifacts/eval_unpaired.json @@ -0,0 +1,322 @@ +{ + "split": "test", + "samples": 1000, + "bridge_mode": "unpaired_gw", + "retrieval": { + "i2t_r@1": 0.0, + "t2i_r@1": 0.0020000000949949026, + "i2t_r@5": 0.004999999888241291, + "t2i_r@5": 0.004999999888241291, + "i2t_r@10": 0.00800000037997961, + "t2i_r@10": 0.010999999940395355, + "i2t_median_rank": 465.0, + "t2i_median_rank": 482.0, + "chance_r@1": 0.001 + }, + "generation": { + "samples": 100, + "mean_best_reference_unigram_f1": 0.22015821592727178, + "examples": [ + { + "row": 25, + "generated": "A man with a large brown jacket and a large brown hat is sitting on a stool. \nHe is wearing a large brown jacket and a large brown hat.", + "references": [ + "The man with pierced ears is wearing glasses and an orange hat.", + "A man with glasses is wearing a beer can crocheted hat.", + "A man with gauges and glasses is wearing a Blitz hat.", + "A man in an orange hat starring at something.", + "A man wears an orange hat and glasses." + ], + "unigram_f1": 0.4 + }, + { + "row": 34, + "generated": "Nicobos is getting a kick in the back of the head. Comments on the video. Commenting on the video. Commenting on the video. Comment", + "references": [ + "A black and white dog is running in a grassy garden surrounded by a white fence.", + "A Boston Terrier is running on lush green grass in front of a white fence.", + "A black and white dog is running through the grass.", + "A dog runs on the green grass near a wooden fence.", + "A Boston terrier is running in the grass." + ], + "unigram_f1": 0.25641025641025644 + }, + { + "row": 51, + "generated": "A man riding a bike on a road for work.  The road is clear of traffic.  The man is walking on the road.  The road is", + "references": [ + "A young female student performing a downward kick to break a board held by her Karate instructor.", + "Girl about to kick a piece of wood in half while karate instructor holds it", + "A girl kicking a stick that a man is holding in tae kwon do class.", + "A girl in karate uniform breaking a stick with a front kick.", + "A girl breaking boards by using karate." + ], + "unigram_f1": 0.2439024390243902 + }, + { + "row": 89, + "generated": "Beaded dogs are running across the grass. \nOptions:\n (A). Grassy\n (B). Wet\n (C). Grassy\n (D", + "references": [ + "Five snowmobile riders all wearing helmets and goggles line up in a snowy clearing in a forest in front of their snowmobiles; they are all wearing black snow pants and from left to right they are wearing a black coat, white coat, red coat, blue coat, and black coat.", + "Five people wearing winter jackets and helmets stand in the snow, with snowmobiles in the background.", + "Five people wearing winter clothing, helmets, and ski goggles stand outside in the snow.", + "A group of snowmobile riders gather in the snow.", + "Group gathered to go snowmobiling." + ], + "unigram_f1": 0.16666666666666669 + }, + { + "row": 97, + "generated": "Many children are playing a game in the courtyard. \r\nand the adults are watching. \r\nand the children are having a good time. \r\nand the adults are", + "references": [ + "Two men sitting on the roof of a house while another one stands on a ladder.", + "Two men on a rooftop while another man stands atop a ladder watching them", + "Three men, one on a ladder, work on a roof.", + "People are fixing the roof of a house.", + "Three men are working on a roof." + ], + "unigram_f1": 0.1764705882352941 + }, + { + "row": 124, + "generated": "The boy is walking in the street while a man is standing on a street corner near a busy market. A man in a blue shirt is reading a newspaper while", + "references": [ + "A bride in a light pink dress poses for a picture with male relatives and is being photographed by a man in a cream shirt with white pants.", + "A man in light colored clothing photographs a group of men wearing dark suits and hats standing around a woman dressed in a strapless gown.", + "A photographer takes a picture of a group of one girl in a pink dress and 10 boys in suits and hats.", + "A man photographs a woman in a pink dress and a throng of men in suits.", + "Man taking a photograph of a well dressed group of teens." + ], + "unigram_f1": 0.3389830508474576 + }, + { + "row": 133, + "generated": "The dog runs through the trees.译为：树丛中的狗奔跑。 二、词组翻译 1. 拥抱译为：", + "references": [ + "A group of 11 people in winter wear such as beanies, skiing jackets, gloves and backpacks are standing in snow paddles outside a house made of ice blocks while a person in front of the door seems to be leading them.", + "A group of people wearing snowshoes, and dressed for winter hiking, is standing in front of a building that looks like it's made of blocks of ice.", + "The people are quietly listening while the story of the ice cabin was explained to them.", + "A group of people standing in front of an igloo.", + "Several students waiting outside an igloo." + ], + "unigram_f1": 0.17391304347826086 + }, + { + "row": 134, + "generated": "Soccer player wearing red uniform is chasing the ball out of the crossbar. There is a ball in the air and the soccer player is behind the ball.", + "references": [ + "A baseball player in red leaps into the air to avoid a player in blue, who is shortly behind him as he catches the ball, while other blue team members look on in the background.", + "A boy in a red uniform is attempting to avoid getting out at home plate, while the catcher in the blue uniform is attempting to catch him.", + "A baseball player coming into home plate with knee held high and the catcher right in front of him waiting for the ball.", + "A baseball player attempting to score a run before the catcher tags the player out.", + "A baseball catcher trying to tag a base runner in a baseball game." + ], + "unigram_f1": 0.38095238095238093 + }, + { + "row": 137, + "generated": "Two children play in the midst of each other.

\n

1. 2. 3. 4. 5.", + "references": [ + "A man wearing a hat and a white shirt is cleaning windows.", + "A man in a white shirt stands high up on scaffolding", + "A man stands on boards on top of a huge ladder.", + "Man works on top of scaffolding.", + "A guy works on a building." + ], + "unigram_f1": 0.08695652173913045 + }, + { + "row": 199, + "generated": "A woman in a white dress standing on a brick wall, with a white umbrella in her hand, and a man in a black suit standing on the ground in", + "references": [ + "A man wearing a reflective vest sits on the sidewalk and holds up pamphlets with bicycles on the cover.", + "Man in bright yellow vest displays bicycle safety information on street.", + "A man and woman talking are interrupted and a man covers his face.", + "A man in a vest is sitting in a chair and holding magazines.", + "A construction worker holds a sign about bicycling." + ], + "unigram_f1": 0.326530612244898 + }, + { + "row": 219, + "generated": "Young rider in a red and white riding a bicycle in a park. \n \nThe park is a large area with trees and grass. \nThe rider is on", + "references": [ + "A mother decides to take her child on a piggyback ride outside their apartment complex.", + "A baby boy in a blue and white striped shirt is sitting on his mother's shoulders.", + "A mother and her young song enjoying a beautiful day outside.", + "A young woman is giving a baby a ride on her shoulders.", + "A woman gives a small child a piggyback ride." + ], + "unigram_f1": 0.3181818181818182 + }, + { + "row": 222, + "generated": "Low speed cars in red and yellow work in the middle of the road in a modern city. They are using a computer to read the map on the side", + "references": [ + "A male volleyball player in red trunks tries to block the shot of the opposing player during a professional volleyball game.", + "Men playing volleyball, with one player missing the ball but hands still in the air.", + "A group of spectators watch a men's sand volleyball game.", + "Volleyball players compete on a sunny day at the beach.", + "Men playing volleyball in the sand." + ], + "unigram_f1": 0.3137254901960784 + }, + { + "row": 228, + "generated": " A pregnant woman with dark brown hair has light brown hair and a red lipstick on her lips.", + "references": [ + "A woman wearing a pink shirt and red apron stands in her restaurant holding food and surrounded by bagged breads, a microwave and the chalkboard menu up on the wall.", + "Woman wearing a yellow hat, pink shirt, and red apron is holding food in a kitchen.", + "Employee in pink shirt smiles for camera inside a restaurant kitchen.", + "A woman holding a bowl of food in a kitchen.", + "Women in store happily holding some food." + ], + "unigram_f1": 0.29411764705882354 + }, + { + "row": 246, + "generated": "White, curly-haired, medium to large bosom, wearing a black top, black pants, and a black shirt. Attire is not specified. Oft", + "references": [ + "An Asian man in spectacles and a t-shirt preparing some food.", + "A man working diligently while sitting at a table.", + "Man sitting using tool at a table in his home.", + "Indian man whittles in his own home.", + "A man is sharpening a knife." + ], + "unigram_f1": 0.21428571428571427 + }, + { + "row": 259, + "generated": "People buy food and drinks from a wide variety of countries. Eating is popular here. A sign may be visible. People often buy food and drinks", + "references": [ + "Spelunkers pose inside a rock cavern while bathed in sunlight from the surface.", + "Shaft of light in a cave shows three spelunkers", + "Three rock climbers are posing on big rocks.", + "People stand inside a rock dome.", + "Three people sit in a cave." + ], + "unigram_f1": 0.125 + }, + { + "row": 279, + "generated": "Woman walking on the shore. I'm looking at a boat. I'm looking at a boat. 2000000000", + "references": [ + "A girl in a polka dotted blue jean dress walks barefoot on a balance beam.", + "A girl in a jean dress is walking along a raised balance beam.", + "The little girl is walking a beam in gymnastics class.", + "A little girl balances on a gymnastics beam.", + "A little girl is walking on a balance beam." + ], + "unigram_f1": 0.2962962962962963 + }, + { + "row": 304, + "generated": "Many people in front of many computers in an American factory. They are discussing important business matters. **Photo Credit: Wikipedia** The United States Department of Commerce is", + "references": [ + "A couple is sitting on the sand with their feet in the water, and they are shaking hands.", + "A man and woman wearing sunglasses sit halfway in the water.", + "A boy and a girl holding hands while sitting on a shore.", + "A couple with sunglasses are laying on the beach.", + "A blond holding hands with a guy in the sand." + ], + "unigram_f1": 0.22222222222222224 + }, + { + "row": 318, + "generated": "-a child with a purple headband wearing a pink shirt and brown shoes has a small red dot on her head. The child is wearing a yellow dress and a", + "references": [ + "A woman standing with 3 other people in a store with two tables, some shelves with coffee and tea for sale, and a refrigerated drink case.", + "A woman in a gray sweater and black baseball cap is standing in line at a shop.", + "A woman in a hat waits to be served at a store.", + "Young people are standing in the store.", + "A woman in a cap at a coffee shop." + ], + "unigram_f1": 0.21428571428571427 + }, + { + "row": 340, + "generated": "Black and brown dogs play with a rock on the ground. (Note: the dogs are brown.) True False Question: Is the dog's fur brown? Answer", + "references": [ + "The person has a striped shirt on and is holding on to a rope on a mountain.", + "A kid rock climbing against the backdrop of a green valley", + "The person in the striped shirt is mountain climbing.", + "A woman in a striped shirt climbs up a mountain.", + "A young man climbs a rocky hill." + ], + "unigram_f1": 0.23255813953488372 + }, + { + "row": 395, + "generated": "A little girl sits on a chair and holds a small object in his/her hand. He/she is looking at a picture on the wall.", + "references": [ + "Two young attractive women in front of a store drop coins for street performers.", + "Two women enjoy the entertainment of street performers.", + "Two men pretend to be statutes while women look on.", + "Two young women stop and watch street performers.", + "Pedestrians interact with street artists." + ], + "unigram_f1": 0.1 + }, + { + "row": 434, + "generated": "A group of people in a crowd are looking at a chaotic action in the background, with a child in the foreground.", + "references": [ + "A group of people standing on the lawn in front of a building.", + "Many people in blue jeans stand in front of a white church.", + "A large group of people stand outside of a church.", + "Family members standing outside a home.", + "People standing outside of a building." + ], + "unigram_f1": 0.39999999999999997 + }, + { + "row": 472, + "generated": "Musketeer jumps in midair as he lands on the water. Overtakes another rider. Shoots into the air. Shows off his speed.", + "references": [ + "A young girl wearing a blue shirt marching in a band playing a trumpet", + "Girl wearing blue shirt and black shorts plays trumped outside.", + "A teenager plays her trumpet on the field at a game.", + "A girl playing trumpet in a marching band.", + "Girl playing the trumpet in a marching band" + ], + "unigram_f1": 0.13793103448275862 + }, + { + "row": 487, + "generated": "Several people walking down a street with a large number of vehicles parked along the side of the street. The street is mostly empty with only a few vehicles parked.", + "references": [ + "Person in a brown sweatshirt does a flip in midair over a trampoline with a river and electrical lines in the background.", + "A young adult is doing a back flip on a trampoline near a lake.", + "A woman is doing flips on a trampoline in front of a lake.", + "A woman does a somersault on a trampoline on the beach.", + "Trampolines are a fun way to exercise." + ], + "unigram_f1": 0.2325581395348837 + }, + { + "row": 520, + "generated": "-蛋, a child wearing a blue jacket and blue pants, is wearing a hat, a scarf, and a pair of boots. She is wearing a blue", + "references": [ + "A man standing in a bar with a neon beer sign hanging on the wall", + "A man stands next to three video game machines and a beer sign.", + "A man stands next to a bank of computer gambling machines.", + "A man is standing by a group of video games in a bar.", + "A man stands next to three video machines." + ], + "unigram_f1": 0.2631578947368421 + }, + { + "row": 539, + "generated": "Two men, one wearing a green hat and the other wearing a blue hat, at a street market.", + "references": [ + "A woman in a green tank top looking at a drill while a crowd looks on.", + "A woman using a power drill in front of a group of people.", + "A woman uses a drill while another man takes her picture.", + "A woman, in a teal tank top, holding a power drill.", + "Spectators about to watch a woman use a drill." + ], + "unigram_f1": 0.29411764705882354 + } + ] + } +} \ No newline at end of file -- cgit v1.2.3