From 6000f1a1228e15b34f1b2354e44bce7a01871e74 Mon Sep 17 00:00:00 2001 From: PJEstrada Date: Wed, 10 May 2023 11:28:56 -0600 Subject: [PATCH 1/3] feat: add import from text data (#61) Co-authored-by: Pablo --- sdk/diffgram/file/file_constructor.py | 42 ++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/sdk/diffgram/file/file_constructor.py b/sdk/diffgram/file/file_constructor.py index 1b14f33..031c673 100644 --- a/sdk/diffgram/file/file_constructor.py +++ b/sdk/diffgram/file/file_constructor.py @@ -111,6 +111,7 @@ def __build_packet_payload(self, file_name: str = None, blob_path: str = None, parent_file_id: int = None, + text_data: str = None, ordinal: int = 0): packet = {'media': {}} packet['media']['url'] = url @@ -119,6 +120,7 @@ def __build_packet_payload(self, # Existing Instances packet['frame_packet_map'] = frame_packet_map packet['ordinal'] = ordinal + packet['text_data'] = text_data packet['type'] = type packet['connection_id'] = connection_id packet['parent_file_id'] = parent_file_id @@ -173,12 +175,50 @@ def from_blob_path(self, file_name = name, directory_id = directory_id, type = "from_blob_path", - parent_file_id=parent_file_id, + parent_file_id = parent_file_id, ordinal = ordinal ) self.from_packet(packet = packet) return True + def from_text_data(self, + file_name: str, + text_data: str, + media_type: str = 'text', + instance_list: list = None, + frame_packet_map: dict = None, + parent_file_id: int = None, + directory_id: int = None, + ordinal: int = 0): + """ + Add the given text as a text File in Diffgram + :param text_data: the raw text to create the text file from. + :param media_type: + :param instance_list: + :param frame_packet_map: + :param parent_file_id: the parent file ID (for compound files) + :param ordinal: the ordinal for the child file (for compound files) + :return: + """ + if self.client.default_directory is None and directory_id is None: + raise Exception("Directory not set. call set_default_directory() to set upload directory.") + if directory_id is None: + directory_id = self.client.directory_id + name = file_name + packet = self.__build_packet_payload( + media_type = media_type, + instance_list = instance_list, + frame_packet_map = frame_packet_map, + file_name = name, + directory_id = directory_id, + type = "from_text_data", + parent_file_id = parent_file_id, + ordinal = ordinal, + text_data = text_data + ) + self.from_packet(packet = packet) + return True + def from_url( self, url: str, From cde9adad4d0fdf5d5494ee29732ed17e9d6d4d63 Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Thu, 22 Jun 2023 18:40:31 -0700 Subject: [PATCH 2/3] fix str int --- sdk/diffgram/job/job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/diffgram/job/job.py b/sdk/diffgram/job/job.py index ed39ba1..940dc35 100644 --- a/sdk/diffgram/job/job.py +++ b/sdk/diffgram/job/job.py @@ -144,7 +144,7 @@ def attach_output_dir(self, dir: Directory, action = 'copy'): data = { 'job_id': self.id, - 'output_dir': str(dir.id), + 'output_dir': dir.id, 'output_dir_action': action } endpoint = "/api/v1/project/{}/job/set-output-dir".format(self.client.project_string_id) From 9ca0983b9bbf32d18d0c65bb798f3c2bb2fc16e1 Mon Sep 17 00:00:00 2001 From: Anthony Sarkis <18080164+anthony-sarkis@users.noreply.github.com> Date: Fri, 14 Jul 2023 16:52:34 -0700 Subject: [PATCH 3/3] Move requirements to be optional --- sdk/diffgram/brain/brain.py | 28 +++++++++---------- .../core/diffgram_dataset_iterator.py | 6 ++-- sdk/requirements.txt | 7 ----- sdk/setup.py | 18 ++++++++---- 4 files changed, 29 insertions(+), 30 deletions(-) diff --git a/sdk/diffgram/brain/brain.py b/sdk/diffgram/brain/brain.py index f1d0eb0..6360dde 100644 --- a/sdk/diffgram/brain/brain.py +++ b/sdk/diffgram/brain/brain.py @@ -2,21 +2,7 @@ import tempfile -try: - import cv2 -except: - print("Warning: Could not import cv2. Some SDK functions may not be available.") - -try: - import tensorflow as tf -except: - print("Warning: Could not import tensorflow. Some SDK functions may not be available") - -import numpy as np import requests -import scipy.misc - -import diffgram.utils.visualization_utils as vis_util class Brain(): @@ -37,6 +23,20 @@ def __init__( if local is true will perform additional setup work local_setup() """ + try: + import cv2 + except: + print("Info: Could not import cv2. Some SDK functions may not be available.") + + try: + import tensorflow as tf + except: + print("Info: Could not import tensorflow. Some SDK functions may not be available") + + import numpy as np + import scipy.misc + import diffgram.utils.visualization_utils as vis_util + self.client = client diff --git a/sdk/diffgram/core/diffgram_dataset_iterator.py b/sdk/diffgram/core/diffgram_dataset_iterator.py index b702e5e..4bccda0 100644 --- a/sdk/diffgram/core/diffgram_dataset_iterator.py +++ b/sdk/diffgram/core/diffgram_dataset_iterator.py @@ -1,8 +1,5 @@ import logging -from PIL import Image, ImageDraw -from imageio import imread -import numpy as np import traceback import sys from threading import Thread @@ -137,6 +134,7 @@ def set_custom_url_signer(self, signer_fn: Callable): self.custom_signer_fn = signer_fn def get_image_data(self, diffgram_file): + from imageio import imread MAX_RETRIES = 10 image = None if hasattr(diffgram_file, 'image'): @@ -254,6 +252,8 @@ def get_file_instances(self, diffgram_file) -> dict: return sample def extract_masks_from_polygon(self, instance_list, diffgram_file, empty_value = 0): + from PIL import Image, ImageDraw + import numpy as np nx, ny = diffgram_file.image['width'], diffgram_file.image['height'] mask_list = [] if nx is None or ny is None: diff --git a/sdk/requirements.txt b/sdk/requirements.txt index 651c49d..7357e89 100644 --- a/sdk/requirements.txt +++ b/sdk/requirements.txt @@ -1,8 +1 @@ requests>=2.20.1 -opencv-python>=4.0.0.21 -scipy>=1.1.0 -six>=1.9.0 -tensorflow>=1.12.0 -pillow -torch -imageio>=2.9.0 \ No newline at end of file diff --git a/sdk/setup.py b/sdk/setup.py index 423dc37..a9c7c77 100644 --- a/sdk/setup.py +++ b/sdk/setup.py @@ -24,10 +24,16 @@ "Operating System :: OS Independent", ], install_requires=[ - 'requests>=2.20.1', - 'scipy>=1.1.0', - 'six>=1.9.0', - 'pillow>=6.1.0', - 'imageio>=2.9.0' - ] + 'requests>=2.20.1' + ], + extras_require = { + 'ml': ["opencv-python>=4.0.0.21", + 'pillow>=6.1.0', + 'scipy>=1.1.0', + 'pillow>=6.1.0', + 'imageio>=2.9.0', + 'tensorflow>=1.12.0', + 'torch' + ] + } ) \ No newline at end of file