updated model to use text-davinci-003

README.md

@@ -20,12 +20,9 @@ Below is an interactive demo for the simulated tabletop manipulation domain, see
 
 ## Preparations
 1. Obtain an [OpenAI API Key](https://openai.com/blog/openai-api/)
-2. Gain Codex access by [joining the waitlist](https://openai.com/blog/openai-codex/) 
-
-Once you have Codex access you can use code-davinci-002. Using the GPT-3 model (text-dainvci-002) is also ok, but performance won't be as good (there will be more code logic errors).
 
 ## Usage 
-1. Fill in the API Key, model name, and how many blocks and bowls to be spawned in the environment. 
+1. Fill in the API Key and how many blocks and bowls to be spawned in the environment. 
 2. Click Setup/Reset Simulation
 3. Based on the new randomly sampled object names, input an instruction and click Run Instruction. If successful, this will render a video and update the simulation environment visualization.
 
@@ -54,4 +51,3 @@ Note object names may need to be changed depending the sampled object names.
 * Prompt saturation - if too many instructions (10+) are executed in a row, then the LLM may start to ignore examples in the early parts of the prompt.
 * Ambiguous instructions - if a given instruction doesn't lead to the desired actions, try rephrasing it to remove ambiguities (e.g. place the block on the closest bowl -> place the block on its closest bowl)
 * Maximum token length - you may hit the maximum token length if running multiple commands in sequence. Please reset the simulation when this happens.
-* If you get the error "That model does not exist" it means you're trying to use Codex w/o Codex access. Switch to text-davinci-002 instead.

app.py

@@ -20,7 +20,6 @@ class DemoRunner:
     def __init__(self):
         self._cfg = OmegaConf.to_container(OmegaConf.load('cfg.yaml'), resolve=True)
         self._env = None
-        self._model_name = ''
         self._md_logger = MarkdownLogger()
 
     def make_LMP(self, env):
@@ -30,8 +29,6 @@ class DemoRunner:
             'init_objs': list(env.obj_name_to_id.keys()),
             'coords': cfg['tabletop_coords']
         }
-        for vs in cfg['lmps'].values():
-            vs['engine'] = self._model_name
 
         LMP_env = LMP_wrapper(env, cfg)
         # creating APIs that the LMPs can interact with
@@ -68,9 +65,8 @@ class DemoRunner:
 
         return lmp_tabletop_ui
 
-    def setup(self, api_key, model_name, n_blocks, n_bowls):
+    def setup(self, api_key, n_blocks, n_bowls):
         openai.api_key = api_key
-        self._model_name = model_name
 
         self._env = PickPlaceEnv(render=True, high_res=True, high_frame_rate=False)
         list_idxs = np.random.choice(len(ALL_BLOCKS), size=max(n_blocks, n_bowls), replace=False)
@@ -107,7 +103,7 @@ class DemoRunner:
         return self._md_logger.get_log(), self._env.get_camera_image(), video_file_name
 
 
-def setup(api_key, model_name, n_blocks, n_bowls):
+def setup(api_key, n_blocks, n_bowls):
     if not api_key:
         return 'Please enter your OpenAI API key!', None, None
     
@@ -116,7 +112,7 @@ def setup(api_key, model_name, n_blocks, n_bowls):
 
     demo_runner = DemoRunner()
     
-    info, img = demo_runner.setup(api_key, model_name, n_blocks, n_bowls)
+    info, img = demo_runner.setup(api_key, n_blocks, n_bowls)
     return info, img, demo_runner
 
 
@@ -141,7 +137,6 @@ if __name__ == '__main__':
             with gr.Column():
                 with gr.Row():
                     inp_api_key = gr.Textbox(label='OpenAI API Key (this is not stored anywhere)', lines=1)
-                    inp_model_name = gr.Dropdown(label='Model Name', choices=['code-davinci-002', 'text-davinci-002'], value='code-davinci-002')
                 with gr.Row():
                     inp_n_blocks = gr.Slider(label='Number of Blocks', minimum=0, maximum=4, value=3, step=1)
                     inp_n_bowls = gr.Slider(label='Number of Bowls', minimum=0, maximum=4, value=3, step=1)
@@ -162,7 +157,7 @@ if __name__ == '__main__':
         
         btn_setup.click(
             setup, 
-            inputs=[inp_api_key, inp_model_name, inp_n_blocks, inp_n_bowls], 
+            inputs=[inp_api_key, inp_n_blocks, inp_n_bowls], 
             outputs=[info_setup, img_setup, state]
         )
         btn_run.click(

cfg.yaml

@@ -2,7 +2,7 @@
 lmps:
   tabletop_ui:
     prompt_path: prompts/tabletop_ui.py
-    engine: model_name
+    engine: text-davinci-003
     max_tokens: 256
     temperature: 0
     query_prefix: '# '
@@ -15,7 +15,7 @@ lmps:
     return_val_name: ret_val
   parse_obj_name:
     prompt_path: prompts/parse_obj_name.py
-    engine: model_name
+    engine: text-davinci-003
     max_tokens: 512
     temperature: 0
     query_prefix: '# '
@@ -28,7 +28,7 @@ lmps:
     return_val_name: ret_val
   parse_position:
     prompt_path: prompts/parse_position.py
-    engine: model_name
+    engine: text-davinci-003
     max_tokens: 512
     temperature: 0
     query_prefix: '# '
@@ -41,7 +41,7 @@ lmps:
     return_val_name: ret_val
   parse_question:
     prompt_path: prompts/parse_question.py
-    engine: model_name
+    engine: text-davinci-003
     max_tokens: 512
     temperature: 0
     query_prefix: '# '
@@ -54,7 +54,7 @@ lmps:
     return_val_name: ret_val
   transform_shape_pts:
     prompt_path: prompts/transform_shape_pts.py
-    engine: model_name
+    engine: text-davinci-003
     max_tokens: 512
     temperature: 0
     query_prefix: '# '
@@ -67,7 +67,7 @@ lmps:
     return_val_name: new_shape_pts
   fgen:
     prompt_path: prompts/fgen.py
-    engine: model_name
+    engine: text-davinci-003
     max_tokens: 512
     temperature: 0
     query_prefix: '# define function: '

prompts/tabletop_ui.py

@@ -4,7 +4,7 @@ from env_utils import put_first_on_second, get_obj_pos, get_obj_names, say, get_
 from plan_utils import parse_obj_name, parse_position, parse_question, transform_shape_pts
 
 objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bowl', 'green bowl']
-# the yellow block on the yellow bowl.
+# place the yellow block on the yellow bowl.
 say('Ok - putting the yellow block on the yellow bowl')
 put_first_on_second('yellow block', 'yellow bowl')
 objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bowl', 'green bowl']
@@ -15,6 +15,11 @@ objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bow
 say('Got it - putting the green block on the top right corner')
 corner_pos = parse_position('top right corner')
 put_first_on_second('green block', corner_pos)
+objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bowl', 'green bowl']
+# stack the blue bowl on the yellow bowl on the green block.
+order_bottom_to_top = ['green block', 'yellow block', 'blue bowl']
+say(f'Sure - stacking from top to bottom: {", ".join(order_bottom_to_top)}')
+stack_objects_in_order(object_names=order_bottom_to_top)
 objects = ['cyan block', 'white block', 'cyan bowl', 'blue block', 'blue bowl', 'white bowl']
 # move the cyan block into its corresponding bowl.
 matches = {'cyan block': 'cyan bowl'}
@@ -34,7 +39,7 @@ say('Sure thing - putting the yellow block between the blue bowl and the green b
 target_pos = parse_position('a point in the middle betweeen the blue bowl and the green block')
 put_first_on_second('yellow block', target_pos)
 objects = ['yellow block', 'red block', 'yellow bowl', 'gray block', 'gray bowl', 'red bowl']
-# can you cut the blocks in half.
+# can you cut the bowls in half.
 say('no, I can only move objects around')
 objects = ['yellow block', 'green block', 'yellow bowl', 'gray block', 'gray bowl', 'green bowl']
 # stack the blocks on the right side with the gray one on the bottom.
@@ -43,24 +48,24 @@ right_side = parse_position('the right side')
 put_first_on_second('gray block', right_side)
 order_bottom_to_top = ['gray block', 'green block', 'yellow block']
 stack_objects_in_order(object_names=order_bottom_to_top)
+objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bowl', 'green bowl']
+# hide the blue bowl.
+bowl_name = np.random.choice(['yellow bowl', 'green bowl'])
+say(f'Sounds good! Hiding the blue bowl under the {bowl_name}')
+put_first_on_second(bowl_name, 'blue bowl')
 objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
-# stack everything with the green block on top.
-say('Ok! Stacking everything with the green block on the top')
-order_bottom_to_top = ['blue bowl', 'pink bowl', 'green bowl', 'pink block', 'blue block', 'green block']
-stack_objects_in_order(object_names=order_bottom_to_top)
-objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
-# move the grass-colored block to the left.
-say('Sure - moving the green block left by 10 centimeters')
-left_pos = parse_position('a point 10cm left of the green block')
-put_first_on_second('green block', left_pos)
+# move the grass-colored bowl to the left.
+say('Sure - moving the green bowl left by 10 centimeters')
+left_pos = parse_position('a point 10cm left of the green bowl')
+put_first_on_second('green bowl', left_pos)
 objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
-# why did you move the red block.
-say(f'I did not move the red block')
+# why did you move the red bowl.
+say(f'I did not move the red bowl')
 objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
 # undo that.
-say('Sure - moving the green block right by 10 centimeters')
-left_pos = parse_position('a point 10cm right of the green block')
-put_first_on_second('green block', left_pos)
+say('Sure - moving the green bowl right by 10 centimeters')
+left_pos = parse_position('a point 10cm right of the green bowl')
+put_first_on_second('green bowl', left_pos)
 objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
 # place the top most block to the corner closest to the bottom most block.
 top_block_name = parse_obj_name('top most block', f'objects = {get_obj_names()}')
@@ -69,13 +74,22 @@ closest_corner_pos = parse_position(f'the corner closest to the {bottom_block_na
 say(f'Putting the {top_block_name} on the {get_corner_name(closest_corner_pos)}')
 put_first_on_second(top_block_name, closest_corner_pos)
 objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
-# move the brown block to the side closest to the green bowl.
-closest_side_position = parse_position('the side closest to the green bowl')
-say(f'Got it - putting the brown block on the {get_side_name(closest_side_position)}')
-put_first_on_second('brown block', closest_side_position)
+# move the brown bowl to the side closest to the green block.
+closest_side_position = parse_position('the side closest to the green block')
+say(f'Got it - putting the brown bowl on the {get_side_name(closest_side_position)}')
+put_first_on_second('brown bowl', closest_side_position)
+objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
+# place the green block to the right of the bowl that has the blue block.
+bowl_name = parse_obj_name('the bowl that has the blue block', f'objects = {get_obj_names()}')
+if bowl_name:
+  target_pos = parse_position(f'a point 10cm to the right of the {bowl_name}')
+  say(f'No problem - placing the green block to the right of the {bowl_name}')
+  put_first_on_second('green block', target_pos)
+else:
+  say('There are no bowls that has the blue block')
 objects = ['brown bowl', 'green block', 'brown block', 'green bowl', 'blue bowl', 'blue block']
 # move the other blocks to the bottom corners.
-block_names = parse_obj_name('blocks other than the green block', f'objects = {get_obj_names()}')
+block_names = parse_obj_name('blocks other than the blue block', f'objects = {get_obj_names()}')
 corners = parse_position('the bottom corners')
 for block_name, pos in zip(block_names, corners):
   put_first_on_second(block_name, pos)
@@ -107,6 +121,12 @@ say(f'Ok - moving the other {len(block_names)} blocks to different corners')
 for block_name, pos in zip(block_names, corners):
   put_first_on_second(block_name, pos)
 objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
+# is the pink block on the green bowl.
+if parse_question('is the pink block on the green bowl', f'objects = {get_obj_names()}'):
+  say('Yes - the pink block is on the green bowl.')
+else:
+  say('No - the pink block is not on the green bowl.')
+objects = ['pink block', 'green block', 'pink bowl', 'blue block', 'blue bowl', 'green bowl']
 # what are the blocks left of the green bowl.
 left_block_names =  parse_question('what are the blocks left of the green bowl', f'objects = {get_obj_names()}')
 if len(left_block_names) > 0:
@@ -121,6 +141,10 @@ objects = ['yellow block', 'green block', 'yellow bowl', 'blue block', 'blue bow
 order_bottom_to_top = ['green bowl', 'blue block', 'green block', 'yellow block']
 say('stacking the blocks on the green bowl')
 stack_objects_in_order(object_names=order_bottom_to_top)
+objects = ['yellow block', 'green block', 'yellow bowl', 'gray block', 'gray bowl', 'green bowl']
+# show me what happens when the desert gets flooded by the ocean.
+say('putting the yellow bowl on the blue bowl')
+put_first_on_second('yellow bowl', 'blue bowl')
 objects = ['pink block', 'gray block', 'orange block']
 # move all blocks 5cm toward the top.
 say('Ok - moving all blocks 5cm toward the top')