{
  "samples": 100,
  "mean_best_reference_unigram_f1": {
    "initial": 0.24513073565703838,
    "final": 0.248555386843474,
    "oracle": 0.5184897529033539
  },
  "energy_protocol": "No cross-modal map and no image-text pair is used by the energy or optimizer. Paired text is loaded only for trajectory and oracle diagnostics; the final step is fixed by CLI arguments.",
  "prefix_training": "text only; no image features or image-text pairs",
  "examples": [
    {
      "row": 67,
      "references": [
        "A group of people stand in the back of a truck filled with cotton.",
        "Men are standing on and about a truck carrying a white substance.",
        "A group of people are standing on a pile of wool in a truck.",
        "A group of men are loading cotton onto a truck",
        "Workers load sheared wool onto a truck."
      ],
      "initial": "Two women make and use small breads.  They are making a necklace.  They are both young. \n A: yes B: no\nYou are",
      "final": "A man wearing a coal miner's uniform is working on the railroad.",
      "oracle": "A group of people are loading a truck with cotton on top of a pile of steel."
    },
    {
      "row": 99,
      "references": [
        "An adult wearing a gray shirt with red sleeves sleeping on a couch.",
        "A woman in black pants napping on the couch.",
        "A man sleeping in a green room on a couch.",
        "The young woman is sleeping in her room.",
        "A long-haired man sleeping on a couch"
      ],
      "initial": "A young boy is playing with a green and yellow pipe in a suburban area.",
      "final": "Black female with blond-hair, wearing red and white, orange and blue, silver and gold, and blue and white, is riding a red and white horse",
      "oracle": "A sleepy woman in a yellow shirt sleeping on a bed."
    },
    {
      "row": 100,
      "references": [
        "Boy in brown shirt with headphones on sits on woman's shoulders in a crowd.",
        "A kid wearing headphones sits on his mom's shoulders while in a crowd.",
        "A boy with headphones on sitting on top of a woman's shoulders.",
        "A child with headphones riding on his mother's shoulders.",
        "A boy wearing headphones sits on a woman's shoulders."
      ],
      "initial": "Two people in a conference room microphone speak to a large audience.  Source: YouTube. 00:00.00.00.0",
      "final": "A child is sitting on a wooden swing in a red hat. The child is wearing a red shirt. \nSubjects: child, hat, red, swing,",
      "oracle": "A boy with headphones on sitting on a girl's head while riding a bike."
    },
    {
      "row": 109,
      "references": [
        "Two men, standing on an ice, looking into something covered with a blue tarp.",
        "Two men are about to enter an ice fishing tent on a snow covered lake.",
        "Two people standing outside a blue tent structure on a snowy surface.",
        "Two men setting up a blue ice fishing hut on an iced over lake",
        "A tent is being set up on the ice."
      ],
      "initial": "A group of men are sitting around a campfire eating food. They all appear to be wearing blue jackets.",
      "final": "Two men are laughing while rolling a ball of grass.",
      "oracle": "Two men are laying on a tent on a frozen lake that is being heated up. \nOptions: (A) on the ground (B) on the ice"
    },
    {
      "row": 118,
      "references": [
        "A man in a green shirt and red life jacket is sitting in a canoe drifting around the lake.",
        "A man wearing a life jacket is in a small boat on a lake with a ferry in view.",
        "A man wearing a red life-vest is sitting in a canoe on a body of water.",
        "A balding man wearing a red life jacket is sitting in a small boat.",
        "A man in a red life vest is riding in a canoe."
      ],
      "initial": "Two people looking over the crowd of people holding a glass of beer.",
      "final": "Five people all wearing hats while sitting in a rectangular table. \n\nWhat is the normal object of the man's attention? \n\nChoose your answer from: (A",
      "oracle": "A man in a blue life jacket is sitting in a raft on a river."
    },
    {
      "row": 161,
      "references": [
        "A woman in jeans and a red coat and carrying a multicolored handbag spreads put her arms while leaping up from a cobblestone street.",
        "A lady in a red coat, holding a bluish hand bag likely of asian descent, jumping off the ground for a snapshot.",
        "A woman in a red jacket, with a blue and gold handbag is jumping while smiling on a cobblestone street.",
        "A woman wearing a red coat is jumping outdoors.",
        "Girl in red jumping for joy."
      ],
      "initial": "A brown dog runs in the forest with his leash. Laborious in the background. License: CC 3.0.0.0.0. License",
      "final": "A kid is falling down and smiling from a snowball. Near a tree.",
      "oracle": "A woman in a red jacket and blue jeans jumping off a bridge."
    },
    {
      "row": 170,
      "references": [
        "Two dogs run towards each other on a rocky area with water in the background.",
        "A brown dog is running after a black dog on a rocky shore.",
        "Two dogs run across stones near a body of water.",
        "A brown dog is running after the black dog.",
        "Two dogs playing on a beach."
      ],
      "initial": "Two women wearing red blankets and sandals are sitting on a wooden structure. They are playing with a fire. \nClassification: Natural. \nClassification: Natural. \n",
      "final": "A woman in a pink dress is watching a white dog pull a blue and yellow wagon across a brown and white field. Women and children are watching a woman in",
      "oracle": "Two dogs are running on a rocky beach next to a water source. Male and female dogs are running together. Scene is in a wooded area."
    },
    {
      "row": 200,
      "references": [
        "A young helmeted man, in his team uniform, is swinging his bat at an incoming baseball.",
        "A young boy wearing a Giants jersey swings a baseball bat at an incoming pitch.",
        "A boy playing for the Giants baseball team swinging at a baseball.",
        "A child swings a baseball bat at the ball.",
        "This little boy hitting a ball."
      ],
      "initial": "A bike rider doing a trick on a bicycle in the background. Motivated to be cool. \n\n\nOutput:\n![bike rider doing trick](https://i.stack",
      "final": "A man on a bridge with a poster of a giant dancer on the left.",
      "oracle": "A young boy swings a baseball at a ball hit by a ball player. The boy is wearing a striped shirt."
    },
    {
      "row": 203,
      "references": [
        "A guy sitting at a desk talking on a phone with books laying everywhere.",
        "Man talking on a phone, surrounded by books in an office.",
        "A man sits at a desk in room cluttered with books.",
        "A man on the phone surrounded by stacks of books.",
        "A man in a cluttered office is using the telephone"
      ],
      "initial": "A brown and white man with a camera and a gray shirt is walking around. He is holding a cellphone. \nAnswer: Step 1: Identify the main",
      "final": "One woman with long brown hair and pink shoes standing with her hands on her hips and another woman with long black hair and blue jeans standing nearby with her hands on",
      "oracle": "A man in a chair at a desk talking on his cellphone. He is standing in a cluttered room.} \n# A man in a chair talking on"
    },
    {
      "row": 257,
      "references": [
        "Woman is enjoying herself being outdoors biking in a forrest preserve.",
        "A girl in an orange tank top is walking her bike through the forrest.",
        "A smiling woman in a peach tank top stands holding a mountain bike",
        "A happy teenage girl taking a break from her bike ride.",
        "A girl smiles while holding her bike in a field."
      ],
      "initial": "A couple in a pink car, with a white and black umbrella on the side.  Neighbors watching.  Pedestrian.  Pink car. ",
      "final": "Closerman in the snow, visible in the foreground. He leaps across the street. Screenshot by Chris P..\n\n\n<a href=\"https://i0",
      "oracle": "A girl in a pink shirt is smiling while riding a bike in a park. Motto: Happy to be a cyclist.  Motto: Happy to be"
    },
    {
      "row": 280,
      "references": [
        "A boy wearing blue and yellow walking on a cliff edge.",
        "A boy in yellow shorts is standing on top of a cliff.",
        "A young child is standing alone on some jagged rocks.",
        "A little boy standing high in the air on a rock.",
        "Child stands near edge of cliff."
      ],
      "initial": "A man sits on a large blanket near a statue of a man with a large hand. \n##### Related Questions:\n\n1. ### Math\n\nA man is standing",
      "final": "A mountain climber stands atop a mountain while he picks up a bag of snow. He is taking a break from the snow-covered mountain and is walking to the",
      "oracle": "A young boy standing on a cliff ledge with blue shorts."
    },
    {
      "row": 282,
      "references": [
        "A snowmobile rider flies through the air on his or her machine in front of tall pine trees.",
        "Rider jumps snowmobile high in rural area.",
        "A person on a snowmobile in mid jump.",
        "A snowmobiler flies through the air.",
        "The man is up in the air."
      ],
      "initial": "A couple with a weird-looking car, a man with a big, orange hat, and a woman with a weird-looking face, all in a window. They",
      "final": "Four men and two women sit at tables in a small boat making a smoothie. Images of colorful fruits and vegetables are shown.   Caption: A smoothie",
      "oracle": "A snowboarder flies high in the air on a mountain. \n;; \nmountain biking near the snow\nExplanation: The given text is a description of"
    },
    {
      "row": 293,
      "references": [
        "A mother and three children collecting garbage from a blue and white garbage can on the street.",
        "A woman and three children are picking something out of a blue trashcan.",
        "Three young children stand around a blue and white barrel.",
        "Four children are standing around a blue and white barrel.",
        "A group of children look into a barrel."
      ],
      "initial": "A blond boy in a blue shirt is in a white box while being introduced to a large white box.",
      "final": "There is a beach ball and a plastic ball that are on the beach. There is water in the air. There is a splash of water on the beach.",
      "oracle": "Three children are looking up at a blue and white container."
    },
    {
      "row": 301,
      "references": [
        "A dark-haired woman in a green and white apron sits amid her outdoor display of flowers for sale on a city street.",
        "A woman is sitting by her dried flower display at an outside market.",
        "A woman sells flowers and incense at the market.",
        "A lady is sitting down tending to her stand.",
        "A woman in an apron shopping at a market."
      ],
      "initial": "A man lays blue and gray on the sand to build a pile of sand. He uses a shovel.  A blue and gray man lays on the sand",
      "final": "A small child is using a shovel to dig a hole.",
      "oracle": "A woman is sitting at a flower stand in the market. She is working on her garden."
    },
    {
      "row": 322,
      "references": [
        "There is a woman and man wearing orange shirts, playing instruments with sheet paper in front of them.",
        "A cello player is playing during a concert.",
        "A woman in a red shirt playing the cello.",
        "A female playing a song on her violin.",
        "A woman in a red shirt plays a cello."
      ],
      "initial": "A player in a red jacket playing with a pipe while other people watch. \nSentence: A man in a red jacket is playing with a pipe while other people",
      "final": "A baseball team playing in a stadium, with a drummer in the front and a sound engineer in the back.",
      "oracle": "A woman playing the cello in a black shirt."
    },
    {
      "row": 370,
      "references": [
        "People on ATVs and dirt bikes are traveling along a worn path in a field surrounded by trees.",
        "Three people on two dirt-bikes and one four-wheeler are riding through brown grass.",
        "Three people ride off-road bikes through a field surrounded by trees.",
        "Three people on motorbikes follow a trail through dry grass.",
        "Three people are riding around on ATV's and motorcycles."
      ],
      "initial": "Two young men in t-shirts ride on ice in front of a crowd of people. Title: Ice Skating in the Snow\n\nCaption: Two young men in",
      "final": "Two men surf in blue water.  The waves crash against the shore.  The sun is shining.  The画面主视觉。Caption: Surfing in",
      "oracle": "Three people are riding on dirt bikes around a dirt road. They are wearing helmets. \nChoices: (a). Yes (b). No;\nWould the answer"
    },
    {
      "row": 377,
      "references": [
        "An older man sits back and relaxes on a patio outside an adobe building where many bicycles are propped.",
        "Topless old man wearing slippers, navy blue pants and a white hat while reclining in a chair outside.",
        "A shirtless man in a white cap relaxes in a deck chair, close to three parked bicycles.",
        "A half naked man is sleeping on his chair outdoors.",
        "Man relaxing in a folding chair on the street."
      ],
      "initial": "The man with a white beard and military-style dress has a white headband and a military-style vest. He is wearing a light jacket and a light hat.",
      "final": "Two blond dogs, one of each breed, are in a competition with a brown dog.  The brown dog is looking at the blond dog.  Both dogs",
      "oracle": "A relaxed man in shorts lounges in a pool of water while lying down on a towel. \n####\nA relaxed man lounges in a pool of water while"
    },
    {
      "row": 382,
      "references": [
        "A group of people are standing outside at a carnival, near a ride called the Green Jungle.",
        "A crowd of people standing next to a shack labeled \"Green Jungle.\"",
        "A group of people standing in front of a hut in a parking lot.",
        "Crowd standing near \"GREEN JUNGLE\" exhibit at festival.",
        "A group of people are gathered."
      ],
      "initial": "A large dog is lying in a field next to a small brown dog. Large dog is next to a small brown dog. \nQuestion: What is the relationship",
      "final": "A small child is running on a sidewalk with a lot of other people.",
      "oracle": "A group of people standing in a crowd of people near a sign."
    },
    {
      "row": 387,
      "references": [
        "A young girl sitting on a small chair surrounded by what looks like handmade fabrics and bags.",
        "A girl wearing a colorful hat sits on a woven mat.",
        "A young woman is making rugs in the rain forest",
        "A woman in a red hat sits on a rug.",
        "A woman sitting in a hut."
      ],
      "initial": "A black dog is cowed by a brown and white dog.",
      "final": "A person with a white shirt and glasses standing up with a smile on their face. They are looking at something in the distance.",
      "oracle": "A woman in a pink scarf sits on a wooden floor. \nA woman sitting on a wooden floor wearing a pink scarf."
    },
    {
      "row": 390,
      "references": [
        "There are three young people, one dark-haired holding a bottled drink, one with head tilted back and finger in her mouth, and one with spiked hair.",
        "Three girls make faces as one takes a drink while they stand in a busy street.",
        "Three young women pose for the camera at a parade.",
        "Three women on the street drinking and posing",
        "The young people party in the streets."
      ],
      "initial": "A baseball player swings to make contact with the catcher as he is out of bounds. (USA Baseball) Category: Basketball, Baseball, Baseball Player. Description:",
      "final": "A little kid is climbing a fountain, while a small child looks down at a blue, plastic, fountain. The boy is using a small, blue, toy",
      "oracle": "Three young women stand in the crowd and take a picture. They are all smiling."
    },
    {
      "row": 419,
      "references": [
        "A group of people gather in a bar, and one man in a black shirt holds his arm and signals to the camera.",
        "In a crowd of young people at a bar, a man stands displaying the rock and roll sign.",
        "A man is making a \"rock\" gesture while standing on a stool in a crowded bar.",
        "A single man in a black t-shirt standing above the crowd at a busy bar.",
        "People enjoying drinks in a crowded bar."
      ],
      "initial": "A little boy in a brown shirt is playing with a chain saw. He is jumping.",
      "final": "A child wearing blue clothing and a red cap is wearing a blue coat.",
      "oracle": "A group of people in a bar drinking in a crowd, one stands on a table."
    },
    {
      "row": 422,
      "references": [
        "Woman and man walking across wooden rope bridge with a caution sign beside it.",
        "A man and a woman are crossing over a rope bridge with greenery all over them.",
        "A man and a woman crossing a suspension bridge in a tropical setting.",
        "A bridge through high green plants, a man and a woman on it.",
        "A man and a woman are walking across a rope bridge."
      ],
      "initial": "A couple is cleaning on the lower tubs, with a blue shirt. Friendly. \"",
      "final": "A boy in a blue shirt and a man in a black shirt stand on the edge of a cliff. The boy in the blue shirt is climbing up the cliff",
      "oracle": "A man and a woman cross a bridge over a river and walk under a bridge. \nAttachment:"
    },
    {
      "row": 462,
      "references": [
        "A man in a gray cap, a pink tank top and short skirt, pink-and-black striped socks, and boots juggles knives in front of a brick building whose sign reads \"The Pump Room\" while onlookers watch.",
        "An male entertainer is wearing women's clothing that consists of a pink tank top, skirt, knee high socks, and a hairnet, performing on a unicycle a juggling knives while a crowd watches.",
        "A man, wearing a short pink skirt, long striped socks, a pink tank top, and a hairnet, is juggling knives to the amusement of onlookers.",
        "A street juggler performing while wearing a colorful outfit.",
        "A man in a skirt is jumping while juggling knives."
      ],
      "initial": "A girl with a pink shirt is sitting in a shopping cart on a table in a store.",
      "final": "A high school student is taking off into the distance on a colorful snowman.",
      "oracle": "A man wearing a striped shirt, shorts, and a hat is performing tricks on a trampoline."
    },
    {
      "row": 478,
      "references": [
        "Woman dressed in a green hat and apron is holding a tray full of drink samples.",
        "An Asian girl in a green hat and apron is serving drinks on a tray.",
        "The waitress offers complimentary tea to the patrons.",
        "A girl hands out Starbucks samples at a street fair.",
        "A Starbucks barista offering up a tray of drinks"
      ],
      "initial": "Three people are wearing sandals and are setting up a lawn chair. The setting is in the middle of the grass. The weather is sunny. The photographer is behind",
      "final": "Three children wear a uniform that includes a backpack, caps, and shoes. All have their faces covered.footwear.\nDoes this next sentence follow, given the",
      "oracle": "A waitress in a white shirt and apron serves coffee to people at a buffet table. The waitress is holding a tray."
    },
    {
      "row": 511,
      "references": [
        "Construction workers standing on top of a piece of machinery.",
        "Two Rubbish men standing on theyre truck loading trash.",
        "A blurry picture of a man in a blue shirt and hard hat.",
        "A man in a white hat stands on top of a truck.",
        "Two people are on top of the bed of a truck."
      ],
      "initial": "A man, in a green jacket, works on a large, square area of black and white tiles. The man is about 6 feet tall.  Description",
      "final": "A group of people gathered together in a circle.",
      "oracle": "A man in a shirt and hat standing on top of a truck."
    }
  ]
}