Merge branch 'master' of github.com:qiuwch/unrealcv

unrealcv · Jun 17, 2016 · f8109b9 · f8109b9
2 parents b6ffb20 + 6fde8b4
commit f8109b9
Show file tree

Hide file tree

Showing 8 changed files with 905 additions and 231 deletions.
diff --git a/Source/UnrealCV/Private/ViewMode.cpp b/Source/UnrealCV/Private/ViewMode.cpp
@@ -46,7 +46,8 @@ void FViewMode::SetCurrentBufferVisualizationMode(FString ViewMode)
 
 	// ToneMapper needs to be disabled
 	Viewport->EngineShowFlags.SetTonemapper(false);
-	Viewport->EngineShowFlags.TemporalAA = 1;
+	// TemporalAA needs to be disabled, or it will contaminate the following frame
+	Viewport->EngineShowFlags.SetTemporalAA(false);
 
 	// A complete list can be found from Engine/Config/BaseEngine.ini, Engine.BufferVisualizationMaterials
 	// TODO: BaseColor is weird, check BUG.
@@ -94,6 +95,9 @@ void FViewMode::Lit()
 	Viewport->EngineShowFlags.SetPostProcessing(true);
 	// ToneMapper needs to be enabled, or the screen will be very dark
 	Viewport->EngineShowFlags.SetTonemapper(true);
+	// TemporalAA needs to be disabled, otherwise the previous frame might contaminate current frame.
+	// Check: https://answers.unrealengine.com/questions/436060/low-quality-screenshot-after-setting-the-actor-pos.html for detail
+	Viewport->EngineShowFlags.SetTemporalAA(false);
 }
 
 void FViewMode::Unlit()
@@ -103,10 +107,9 @@ void FViewMode::Unlit()
 	ApplyViewMode(VMI_Unlit, true, Viewport->EngineShowFlags);
 	Viewport->EngineShowFlags.SetMaterials(false);
 	Viewport->EngineShowFlags.SetVertexColors(false);
-	Viewport->EngineShowFlags.LightFunctions = 0;
-	Viewport->EngineShowFlags.DynamicShadows = 0;
+	Viewport->EngineShowFlags.SetLightFunctions(false);
 	Viewport->EngineShowFlags.SetLighting(false);
-	Viewport->EngineShowFlags.AtmosphericFog = 0;
+	Viewport->EngineShowFlags.SetAtmosphericFog(false);
 }
 
 void FViewMode::DebugMode()

diff --git a/client/examples/faster-rcnn.py b/client/examples/faster-rcnn.py
@@ -35,6 +35,7 @@ def init_caffe(): # TODO: parse args into here
 def plot_image(image, boxes=None, scores=None):
     ax.cla() # Clear axis
     ax.imshow(image, aspect='equal')
+    ax.axis('off')
 
     if boxes != None and scores != None:
         CONF_THRESH = 0.8
@@ -92,15 +93,12 @@ def process_image(filename):
 def message_handler(message):
     print 'Got server message %s' % repr(message)
     if message == 'clicked':
-        onclick(None)
-
-def onclick(event):
-    image = ue4cv.client.request('vget /camera/0/lit')
-    process_image(image)
+        image = ue4cv.client.request('vget /camera/0/lit')
+        process_image(image)
 
 if __name__ == '__main__':
     _L = logging.getLogger('ue4cv')
-    _L.setLevel(logging.DEBUG)
+    _L.setLevel(logging.ERROR)
 
     ue4cv.client.message_handler = message_handler
     ue4cv.client.connect()
@@ -110,7 +108,6 @@ def onclick(event):
     else:
         # Initialize the matplotlib
         fig, ax = plt.subplots()
-        fig.canvas.mpl_connect('button_press_event', onclick)
 
         # Show an empty image
         image = np.zeros((300, 300))

diff --git a/client/examples/generate-images.ipynb b/client/examples/generate-images.ipynb
diff --git a/client/examples/ipynb_util.py b/client/examples/ipynb_util.py
@@ -5,7 +5,6 @@
 sys.path.append('..')
 import ue4cv
 
-
 class Timer(object):
     def __init__(self, name=None):
         self.name = name
@@ -19,17 +18,17 @@ def __exit__(self, type, value, traceback):
         print 'Elapsed: %s' % (time.time() - self.tstart)
 
 class Namedict(object):
+    # d = Namedict(a=1, b=2)
+    # d.a
     def __init__(self, **kwargs):
         self.dic = kwargs
         self.__dict__.update(self.dic)
 
     def __repr__(self):
-        return self.dic
+        return str(self.dic)
 
     def __str__(self):
         return str(self.dic)
-# d = namedict(a=1, b=2)
-# d.a
 
 def read_camera_info(filename):
     with open(filename) as f:
@@ -39,23 +38,27 @@ def read_camera_info(filename):
     for line_id in range(len(lines)):
         line = lines[line_id].strip() # Remove \n at the end
         if line_id % 3 == 0: # filename
-            pass
+            pass # ignore this line
         elif line_id % 3 == 1: # location
             location = [float(v) for v in line.split(' ')]
         elif line_id % 3 == 2: # Rotation
             rotation = [float(v) for v in line.split(' ')]
             camera_pos.append((location, rotation))
     return camera_pos
 
-def render_frame(client, pos):
-    loc = pos[0] # location
-    rot = pos[1] # rotation
-    cmd = 'vset /camera/0/location %.3f %.3f %.3f' % (loc[0], loc[1], loc[2])
-    response = client.request(cmd)
-    assert response == 'ok'
-    cmd = 'vset /camera/0/rotation %.3f %.3f %.3f' % (rot[0], rot[1], rot[2])
-    response = client.request(cmd)
-    assert response == 'ok'
+def render_frame(client, cam_pose=None):
+    ''' If you want to render a frame using current camera position, leave the pos argument to None '''
+    if pos is not None:
+        # Set camera position
+        loc = cam_pose[0] # location
+        rot = cam_pose[1] # rotation
+        cmd = 'vset /camera/0/location %.3f %.3f %.3f' % (loc[0], loc[1], loc[2])
+        response = client.request(cmd)
+        assert response == 'ok'
+        cmd = 'vset /camera/0/rotation %.3f %.3f %.3f' % (rot[0], rot[1], rot[2])
+        response = client.request(cmd)
+        assert response == 'ok'
+
     f = Namedict(
         lit = client.request('vget /camera/0/lit'),
         depth = client.request('vget /camera/0/depth'),
@@ -64,7 +67,6 @@ def render_frame(client, pos):
     )
     return f
 
-# Get the color of each object
 class Color(object):
     regexp = re.compile('\(R=(.*),G=(.*),B=(.*),A=(.*)\)')
     def __init__(self, color_str):
@@ -75,16 +77,8 @@ def __init__(self, color_str):
     def __repr__(self):
         return self.color_str
 
-def plot_rendered_frame(f):
-
-    subplot_image(221, f.lit)
-    subplot_image(222, f.depth)
-    subplot_image(223, f.object_mask)
-    subplot_image(224, f.normal)
-    # plt.subplots_adjust(wspace=0, hspace=0)
-    # plt.tight_layout()
-
-def plot_color(color, title):
+def subplot_color(index, color, title):
+    plt.subplot(index)
     im = np.zeros((100, 100, 4))
     # red = np.array([1.0, 0, 0, 1.0])
     color_array = np.array([color.R, color.G, color.B, color.A]) / 255.0 # repmat
@@ -96,18 +90,19 @@ def plot_color(color, title):
 
 def subplot_image(sub_index, image):
     if isinstance(image, str):
-        image = plt.imread(image)
+        image = imread(image)
     plt.subplot(sub_index)
     plt.imshow(image)
     plt.axis('off')
 
-def plot_object_region(image, object_mask):
+def plot_image_with_mask(image, mask):
+    ''' Mask is a binary matrix, only mask==1 will be plotted '''
     if isinstance(image, str):
-        image = plt.imread(image)
+        image = imread(image)
     subplot_image(121, image)
-    obj_image = image.copy()
-    obj_image[~object_mask] = 0
-    subplot_image(122, obj_image)
+    masked_image = image.copy()
+    masked_image[~mask] = 0
+    subplot_image(122, masked_image)
 
 def plot_object_color(object_list, color_mapping):
     N = len(object_list)
@@ -118,100 +113,70 @@ def plot_object_color(object_list, color_mapping):
         plot_color(color, object_name)
         object_id += 1
 
-def get_color_mapping(client, object_list=None):
-    '''
-    Get the color mapping of this scene
-    '''
-    # Get object list
-    if not object_list:
-        object_list = client.request('vget /objects').split(' ')
-    # Get the color mapping for each object
+def get_color_mapping(client, object_list):
+    ''' Get the color mapping for specified objects '''
     color_mapping = {}
     for objname in object_list:
         color_mapping[objname] = Color(client.request('vget /object/%s/color' % objname))
     return color_mapping
 
-class ObjectInstanceMap(object):
-    def __init__(self, color_object_mask, color_mapping):
-        '''
-        color_object_mask, in which each object is labeled with a unique color
-        color_mapping, maps from object name to a color
-        '''
-        self.color_object_mask = color_object_mask
-        self.color_mapping = color_mapping
-        self.dic_object_instance = {}
-
-        # TODO: Make sure one object is only labeled once
-        objects_names = self.color_mapping.keys()
-        for object_name in objects_names:
-            color = self.color_mapping[object_name]
-            region = self.match_color(self.color_object_mask * 255.999, [color.R, color.G, color.B], tolerance=2)
-            if region.sum() != 0: # Present in the image
-                self.dic_object_instance[object_name] = region
-
-    def print_object_size(self):
-        '''
-        Print the size of each object
-        '''
-        for object_name in self.dic_object_instance.keys():
-            print object_name, self.dic_object_instance[object_name].sum()
-
-    def check_coverage(self):
-        '''
-        Check the portion of labeled image
-        '''
-        marked_region = np.zeros(self.color_object_mask.shape[0:2])
-        for object_name in self.dic_object_instance.keys():
-            marked_region += self.dic_object_instance[object_name]
-        assert(marked_region.max() == 1)
-        print marked_region.sum() / (marked_region.shape[0] * marked_region.shape[1])
-
-    @staticmethod
-    def match_color(color_image, target_color, tolerance=3): # Tolerance is used to solve numerical issue
-        match_region = np.ones(color_image.shape[0:2], dtype=bool)
-        for c in range(3): # Iterate over three channels
-            min_val = target_color[c]-tolerance; max_val = target_color[c]+tolerance
-            channel_region = (color_image[:,:,c] >= min_val) & (color_image[:,:,c] <= max_val)
-            # channel_region = color_image[:,:,c] == target_color[c]
-            match_region &= channel_region
-        return match_region
-
-    def count_object_instance(object_mask_filename):
-        object_mask = plt.imread(object_mask_filename)
-        plt.rcParams['figure.figsize'] = (8.0, 8.0)
-        plt.imshow(object_mask)
-        plt.axis('off')
-
-        dict_instance_mask = {}
-        dict_instance_color = {}
-        object_id = 0
-
-        counted_region = np.zeros(object_mask.shape[0:2], dtype=bool)
-        while (~counted_region).sum() != 0:
-            object_id += 1
-            # Count how many unique objects in this scene?
-            remaining_pixels = object_mask[~counted_region]
-            instance_color = remaining_pixels[0,:]
-
-            instance_mask = (object_mask == instance_color).all(axis=2)
-            counted_region += instance_mask
-
-            object_name = 'object%d' % object_id
-            dict_instance_mask[object_name] = instance_mask
-            dict_instance_color[object_name] = instance_color * 255
-        return [dict_instance_mask, dict_instance_color]
-
-if __name__ == '__main__':
-    ue4cv.client.connect()
-
-    camera_pos = read_camera_info('./realistic_rendering_camera_info.txt')
-    pos = camera_pos[1]
-    f = render_frame(ue4cv.client, pos)
-    color_object_mask = plt.imread(f.object_mask)
-
-    objects = ['WallPiece1_22', 'SM_Shelving_6', 'SM_Couch_1seat_5', 'SM_Frame_39', 'SM_Shelving_7', 'SM_Shelving_8']
-    with Timer():
-        color_mapping = get_color_mapping(ue4cv.client, objects)
-        # color_mapping = get_selected_color_mapping(ue4cv.client)
-    object_instance_map = ObjectInstanceMap(color_object_mask, color_mapping)
-    object_instance_map.check_coverage()
+def generate_objectcatetory_json(scene_objects):
+    # Use http://www.jsoneditoronline.org/ to clean the json
+    # http://jsonformat.com/#jsondataurllabel
+    ''' Get object category from object name, with some manual editing '''
+    print '{'
+    for obj in scene_objects:
+        objtype = obj.replace('SM_', '').split('_')[0].replace('BookLP', 'Book').replace('Wire1', 'Wire')
+        print ' ', repr(obj), ':', repr(objtype), ','
+    print '}'
+
+def match_color(color_image, target_color, tolerance=3): # Tolerance is used to solve numerical issue
+    match_region = np.ones(color_image.shape[0:2], dtype=bool)
+    for c in range(3): # Iterate over three channels
+        min_val = target_color[c]-tolerance; max_val = target_color[c]+tolerance
+        channel_region = (color_image[:,:,c] >= min_val) & (color_image[:,:,c] <= max_val)
+        # channel_region = color_image[:,:,c] == target_color[c]
+        match_region &= channel_region
+    return match_region
+
+def compute_instance_mask(object_mask, color_mapping, objects):
+    if isinstance(object_mask, str):
+        object_mask = misc.imread(object_mask)
+
+    dic_instance_mask = {}
+    for object_name in objects:
+        color = color_mapping[object_name]
+        region = match_color(object_mask, [color.R, color.G, color.B], tolerance=3)
+        if region.sum() != 0: # Present in the image
+            dic_instance_mask[object_name] = region
+    return dic_instance_mask
+
+
+def check_coverage(dic_instance_mask):
+    ''' Check the portion of labeled image '''
+    marked_region = None
+    for object_name in dic_instance_mask.keys():
+        instance_mask = dic_instance_mask[object_name]
+        if marked_region is None:
+            marked_region = np.zeros(instance_mask.shape[0:2])
+        marked_region += instance_mask
+
+    assert(marked_region.max() == 1)
+    if marked_region.max() > 1:
+        print 'There are invalid regions in the labeling'
+    coverage = float(marked_region.sum()) / (marked_region.shape[0] * marked_region.shape[1])
+    print 'Coverage %.2f' % coverage
+    return marked_region
+
+
+def get_category_mask(category):
+    mask = None
+    for obj in image_objects:
+        if dic_objects_category[obj] == category:
+            if mask is None:
+                mask = dic_instance_mask[obj]
+            else:
+                mask += dic_instance_mask[obj]
+    if mask is None:
+        print 'Error: the mask is empty'
+    return mask