Merge pull request #5 from dataiku/features/pagination

Adding pagination and shared folders

CHANGELOG.md

@@ -0,0 +1,8 @@
+# Changelog
+
+## [Version 1.1.1](https://github.com/dataiku/dss-plugin-googledrive/releases/tag/v1.1.1) - Feature release - 2021-04-21
+
+- Gives access to Shared Drives
+- Adds pagination (giving access to directories with more than 100 items)
+- Fix issue with empty root id
+- Dependencies updates (google-api-python-client, httplib2)

Jenkinsfile

@@ -0,0 +1,47 @@
+pipeline {
+   options { disableConcurrentBuilds() }
+   agent { label 'dss-plugin-tests'}
+   environment {
+        PLUGIN_INTEGRATION_TEST_INSTANCE="/home/jenkins-agent/instance_config.json"
+    }
+   stages {
+      stage('Run Unit Tests') {
+         steps {
+            sh 'echo "Running unit tests"'
+            catchError(stageResult: 'FAILURE') {
+            sh """
+               make unit-tests
+               """
+            }
+            sh 'echo "Done with unit tests"'
+         }
+      }
+      stage('Run Integration Tests') {
+         steps {
+            sh 'echo "Running integration tests"'
+            catchError(stageResult: 'FAILURE') {
+            sh """
+               make integration-tests
+               """
+            }
+            sh 'echo "Done with integration tests"'
+         }
+      }
+   }
+   post {
+     always {
+        script {
+           allure([
+                    includeProperties: false,
+                    jdk: '',
+                    properties: [],
+                    reportBuildPolicy: 'ALWAYS',
+                    results: [[path: 'tests/allure_report']]
+            ])
+
+            def status = currentBuild.currentResult
+            sh "file_name=\$(echo ${env.JOB_NAME} | tr '/' '-').status; touch \$file_name; echo \"${env.BUILD_URL};${env.CHANGE_TITLE};${env.CHANGE_AUTHOR};${env.CHANGE_URL};${env.BRANCH_NAME};${status};\" >> $HOME/daily-statuses/\$file_name"
+        }
+     }
+   }
+}

Makefile

@@ -1,7 +1,4 @@
-# Public variable to be set by the user in the Makefile
-TARGET_DSS_VERSION=8.0
-
-# evaluate additional variable
+# Makefile variables set automatically
 plugin_id=`cat plugin.json | python -c "import sys, json; print(str(json.load(sys.stdin)['id']).replace('/',''))"`
 plugin_version=`cat plugin.json | python -c "import sys, json; print(str(json.load(sys.stdin)['version']).replace('/',''))"`
 archive_file_name="dss-plugin-${plugin_id}-${plugin_version}.zip"
@@ -16,28 +13,39 @@ plugin:
 	@mkdir dist
 	@echo "{\"remote_url\":\"${remote_url}\",\"last_commit_id\":\"${last_commit_id}\"}" > release_info.json
 	@git archive -v -9 --format zip -o dist/${archive_file_name} HEAD
+	@zip --delete dist/${archive_file_name} "tests/*"
 	@zip -u dist/${archive_file_name} release_info.json
 	@rm release_info.json
 	@echo "[SUCCESS] Archiving plugin to dist/ folder: Done!"
 
 unit-tests:
-	@echo "[START] Running unit tests..."
+	@echo "Running unit tests..."
+	@( \
+		PYTHON_VERSION=`python3 -V 2>&1 | sed 's/[^0-9]*//g' | cut -c 1,2`; \
+		PYTHON_VERSION_IS_CORRECT=`cat code-env/python/desc.json | python3 -c "import sys, json; print(str($$PYTHON_VERSION) in [x[-2:] for x in json.load(sys.stdin)['acceptedPythonInterpreters']]);"`; \
+		if [ $$PYTHON_VERSION_IS_CORRECT == "False" ]; then echo "Python version $$PYTHON_VERSION is not in acceptedPythonInterpreters"; exit 1; else echo "Python version $$PYTHON_VERSION is in acceptedPythonInterpreters"; fi; \
+	)
 	@( \
+		rm -rf ./env/; \
 		python3 -m venv env/; \
 		source env/bin/activate; \
-		pip3 install --upgrade pip; \
-		pip install --no-cache-dir -r tests/python/requirements.txt; \
+		pip install --upgrade pip;\
+		pip install --no-cache-dir -r tests/python/unit/requirements.txt; \
 		pip install --no-cache-dir -r code-env/python/spec/requirements.txt; \
 		export PYTHONPATH="$(PYTHONPATH):$(PWD)/python-lib"; \
-                pytest -o junit_family=xunit2 --junitxml=unit.xml tests/python/unit || true; \
-		deactivate; \
+		pytest tests/python/unit --alluredir=tests/allure_report || ret=$$?; exit $$ret \
 	)
-	@echo "[SUCCESS] Running unit tests: Done!"
 
 integration-tests:
-	@echo "[START] Running integration tests..."
-	# TODO add integration tests
-	@echo "[SUCCESS] Running integration tests: Done!"
+	@echo "Running integration tests..."
+	@( \
+		rm -rf ./env/; \
+		python3 -m venv env/; \
+		source env/bin/activate; \
+		pip3 install --upgrade pip;\
+		pip install --no-cache-dir -r tests/python/integration/requirements.txt; \
+		pytest tests/python/integration --alluredir=tests/allure_report || ret=$$?; exit $$ret \
+	)
 
 tests: unit-tests integration-tests
 

code-env/python/spec/requirements.txt

@@ -1,5 +1,5 @@
-google-api-python-client==1.12.1
+google-api-python-client==1.12.8
 google-auth==1.21.2
 google-auth-httplib2==0.0.4
-httplib2==0.18.0
+httplib2==0.19.0
 oauth2client==4.1.3

plugin.json

@@ -1,6 +1,6 @@
 {
     "id": "googledrive",
-    "version": "1.1.0",
+    "version": "1.1.1",
     "meta": {
         "label": "Google Drive",
         "description": "Read and write data from/to your Google Drive account",

python-fs-providers/googledrive-googledrive/fs-provider.py

@@ -10,7 +10,6 @@
 from dku_googledrive.googledrive_utils import GoogleDriveUtils as gdu
 from dss_constants import DSSConstants
 from dku_googledrive.session import GoogleDriveSession
-from dataikuapi.utils import DataikuException
 
 try:
     from BytesIO import BytesIO  # for Python 2
@@ -246,23 +245,23 @@ def move(self, from_path, to_path):
                 to_item = self.session.get_item_from_path(os.path.split(full_to_path)[0])
 
                 prev_parents = ','.join(p for p in from_item.get(gdu.PARENTS))
-                self.drive.files().update(
+                self.session.drive.files().update(
                     fileId=gdu.get_id(from_item),
                     addParents=gdu.get_id(to_item),
                     removeParents=prev_parents,
                     fields=gdu.ID_PARENTS_FIELDS,
                 ).execute()
             else:
-                file = self.drive.files().get(fileId=gdu.get_id(from_item)).execute()
+                file = self.session.drive.files().get(fileId=gdu.get_id(from_item)).execute()
                 del file[gdu.ID]
                 file[gdu.NAME] = to_name
-                self.drive.files().update(
+                self.session.drive.files().update(
                     fileId=gdu.get_id(from_item),
                     body=file,
                     fields=gdu.ID_PARENTS_FIELDS,
                 ).execute()
         except HttpError as err:
-            raise DataikuException('Error from Google Drive while moving files: ' + err)
+            raise Exception('Error from Google Drive while moving files: ' + err)
 
         return True
 
@@ -275,7 +274,7 @@ def read(self, path, stream, limit):
         item = self.session.get_item_from_path(full_path)
 
         if item is None:
-            raise DataikuException('Path doesn t exist')
+            raise ValueError('Path doesn t exist')
 
         self.session.googledrive_download(item, stream)
 
@@ -297,4 +296,4 @@ def assert_path_is_not_root(self, path):
         black_list = [None, "", "root"]
         if self.root_id in black_list and path is None or path.strip("/") == "":
             logger.error("Will not delete root directory. root_id={}, path={}".format(self.root_id, path))
-            raise DataikuException("Cannot delete root path")
+            raise ValueError("Cannot delete root path")

python-lib/dku_googledrive/googledrive_utils.py

@@ -1,7 +1,10 @@
 import os
 import string
 from datetime import datetime
-from dataikuapi.utils import DataikuException
+
+
+class GoogleDriveUtilsError(ValueError):
+    pass
 
 
 class GoogleDriveUtils(object):
@@ -130,16 +133,16 @@ def file_size(item):
     def check_path_format(path):
         special_names = [".", ".."]
         if not all(c in string.printable for c in path):
-            raise DataikuException('The path contains non-printable char(s)')
+            raise GoogleDriveUtilsError('The path contains non-printable char(s)')
         for element in path.split('/'):
             if len(element) > 1024:
-                raise DataikuException('An element of the path is longer than the allowed 1024 characters')
+                raise GoogleDriveUtilsError('An element of the path is longer than the allowed 1024 characters')
             if element in special_names:
-                raise DataikuException('Special name "{0}" is not allowed in a box.com path'.format(element))
+                raise GoogleDriveUtilsError('Special name "{0}" is not allowed in a box.com path'.format(element))
             if element.endswith(' '):
-                raise DataikuException('An element of the path contains a trailing space')
+                raise GoogleDriveUtilsError('An element of the path contains a trailing space')
             if element.startswith('.well-known/acme-challenge'):
-                raise DataikuException('An element of the path starts with ".well-known/acme-challenge"')
+                raise GoogleDriveUtilsError('An element of the path starts with ".well-known/acme-challenge"')
 
     @staticmethod
     def query_parents_in(parent_ids, name=None, name_contains=None, trashed=None):
@@ -163,6 +166,6 @@ def query_parents_in(parent_ids, name=None, name_contains=None, trashed=None):
     @staticmethod
     def get_root_id(config):
         root_id = config.get("googledrive_root_id")
-        if root_id is None:
+        if not root_id:
             root_id = GoogleDriveUtils.ROOT_ID
         return root_id

python-lib/dku_googledrive/session.py

@@ -13,7 +13,6 @@
 from googleapiclient.errors import HttpError
 from dku_googledrive.googledrive_utils import GoogleDriveUtils as gdu
 from time import sleep
-from dataikuapi.utils import DataikuException
 from dku_googledrive.memory_cache import MemoryCache
 
 try:
@@ -26,6 +25,10 @@
                     format='googledrive plugin %(levelname)s - %(message)s')
 
 
+class GoogleDriveSessionError(ValueError):
+    pass
+
+
 class GoogleDriveSession():
     """
     Google Drive Session
@@ -49,7 +52,7 @@ def __init__(self, config, plugin_config):
             credentials = ServiceAccountCredentials.from_json_keyfile_dict(credentials_dict, scopes)
             http_auth = credentials.authorize(Http())
         self.root_id = config.get("googledrive_root_id")
-        if self.root_id is None:
+        if not self.root_id:
             self.root_id = gdu.ROOT_ID
         self.max_attempts = 5
         self.root_id = gdu.get_root_id(config)
@@ -113,14 +116,28 @@ def googledrive_list(self, query):
         attempts = 0
         while attempts < self.max_attempts:
             try:
-                request = self.drive.files().list(q=query, fields=gdu.LIST_FIELDS).execute()
-                files = request.get('files', [])
+                files = []
+                kwargs = {
+                    'q': query,
+                    'fields': gdu.LIST_FIELDS,
+                    'includeItemsFromAllDrives': True,
+                    'supportsAllDrives': True
+                }
+                initial_call = True
+                next_page_token = None
+                while initial_call or next_page_token:
+                    initial_call = False
+                    if next_page_token:
+                        kwargs['pageToken'] = next_page_token
+                    response = self.drive.files().list(**kwargs).execute()
+                    files.extend(response.get('files', []))
+                    next_page_token = response.get('nextPageToken')
                 return files
             except HttpError as err:
                 self.handle_googledrive_errors(err, "list")
             attempts = attempts + 1
             logger.info('googledrive_list:attempts={}'.format(attempts))
-        raise DataikuException("Max number of attempts reached in Google Drive directory list operation")
+        raise GoogleDriveSessionError("Max number of attempts reached in Google Drive directory list operation")
 
     def create_directory_from_path(self, path):
         tokens = gdu.split_path(path)
@@ -162,7 +179,7 @@ def googledrive_create(self, body, media_body=None):
                 self.handle_googledrive_errors(err, "create")
             attempts = attempts + 1
             logger.info('googledrive_create:attempts={}'.format(attempts))
-        raise DataikuException("Max number of attempts reached in Google Drive directory create operation")
+        raise GoogleDriveSessionError("Max number of attempts reached in Google Drive directory create operation")
 
     def googledrive_upload(self, filename, file_handle, parent_id=None):
         mime = MimeTypes()
@@ -216,7 +233,7 @@ def googledrive_update(self, file_id, body, media_body=None):
                 self.handle_googledrive_errors(err, "update")
             attempts = attempts + 1
             logger.info('googledrive_update:attempts={}'.format(attempts))
-        raise DataikuException("Max number of attempts reached in Google Drive directory update operation")
+        raise GoogleDriveSessionError("Max number of attempts reached in Google Drive directory update operation")
 
     def googledrive_delete(self, item, parent_id=None):
         attempts = 0
@@ -232,7 +249,7 @@ def googledrive_delete(self, item, parent_id=None):
                 self.handle_googledrive_errors(err, "delete")
             attempts = attempts + 1
             logger.info('googledrive_update:attempts={}'.format(attempts))
-        raise DataikuException("Max number of attempts reached in Google Drive directory delete operation")
+        raise GoogleDriveSessionError("Max number of attempts reached in Google Drive directory delete operation")
 
     def handle_googledrive_errors(self, err, context=""):
         if err.resp.status in [403, 500, 503]:
@@ -241,4 +258,4 @@ def handle_googledrive_errors(self, err, context=""):
             reason = ""
             if err.resp.get('content-type', '').startswith('application/json'):
                 reason = json.loads(err.content).get('error').get('errors')[0].get('reason')
-            raise DataikuException("Googledrive {} error : {}".format(context, reason))
+            raise GoogleDriveSessionError("Googledrive {} error : {}".format(context, reason))

tests/python/integration/pytest.ini

@@ -0,0 +1,2 @@
+[pytest]
+usefixtures = plugin dss_target

tests/python/integration/requirements.txt

@@ -0,0 +1,3 @@
+pytest==6.2.1
+dataiku-api-client
+git+git://github.com/dataiku/dataiku-plugin-tests-utils.git@master#egg=dataiku-plugin-tests-utils

tests/python/integration/test_scenario.py

@@ -0,0 +1,7 @@
+from dku_plugin_test_utils import dss_scenario
+
+TEST_PROJECT_KEY = "PLUGINTESTGOOGLEDRIVE"
+
+
+def test_run_googledrive_directory_pagination(user_dss_clients):
+    dss_scenario.run(user_dss_clients, project_key=TEST_PROJECT_KEY, scenario_id="DirectoryPagination")

tests/python/unit/requirements.txt

@@ -0,0 +1,2 @@
+pytest==6.2.1
+allure-pytest==2.8.29

tests/python/unit/test_metadata.py

@@ -1,16 +1,11 @@
-import os
-import sys
-
-## Add stuff to the path to enable exec outside of DSS
-#plugin_root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-#sys.path.append(os.path.join(plugin_root, 'python-lib'))
-
-#from dku_googledrive.googledrive_helpers import is_directory
+import pytest
 from dku_googledrive.googledrive_utils import GoogleDriveUtils
 
 
-def test_is_directory():
-    not_a_directory = {'mimeType' : "text/csv"}
-    directory = {'mimeType' : "application/vnd.google-apps.folder"}
-    assert GoogleDriveUtils.is_directory(not_a_directory) == False
-    assert GoogleDriveUtils.is_directory(directory) == True
+class TestCommonMethods:
+
+    def test_is_directory(self):
+        not_a_directory = {'mimeType': "text/csv"}
+        directory = {'mimeType': "application/vnd.google-apps.folder"}
+        assert GoogleDriveUtils.is_directory(not_a_directory) == False
+        assert GoogleDriveUtils.is_directory(directory) == True