Merge pull request #195 from bird-house/skip-py27

skip python 2.7 support

.gitignore

@@ -68,6 +68,7 @@ docs/build/
 
 # External Sources
 #src/external
+src/
 
 # tests
 *.log

.travis.yml

@@ -1,8 +1,10 @@
+dist: xenial   # required for Python >= 3.7
 language: python
 os:
   - linux
 python:
-  - "2.7"
+  - "3.6"
+  #- "3.7"
 sudo: false
 install:
   - make install
@@ -12,5 +14,3 @@ before_script:
 script:
   - make testall
   - make pep8
-#after_success:
-# - coveralls

CHANGES.rst

@@ -35,7 +35,7 @@ Dar-es-Salaam Release.
 0.8.1 (2018-04-04)
 ==================
 
-Montréal Release.
+Montreal Release.
 
 * configured require_csrf.
 * using check_csrf_token.

Makefile

@@ -1,4 +1,4 @@
-VERSION := 0.3.17
+VERSION := 0.4.0
 RELEASE := master
 
 # Include custom config if it is available

environment.yml

@@ -4,7 +4,7 @@ channels:
   - birdhouse
   - defaults
 dependencies:
-  - python=2.7
+  - python=3.6
   - setuptools
   - curl
   - pyopenssl
@@ -17,12 +17,11 @@ dependencies:
 # ipython used by make passwd
   - ipython
 # phoenix
-  - six
-  - nginx=1.10
+  - nginx
   - supervisor=4
-  - gunicorn=19
+  - gunicorn
   - requests-oauthlib
-  - gevent=1.2
+  - gevent
   - lxml
   - mako
   - dateparser
@@ -55,11 +54,12 @@ dependencies:
       - pyramid-celery==3.0.0
       - pyramid-mailer
       - pyramid-storage
-      - pyramid-twitcher==0.3.8
+      #- pyramid-twitcher==0.4.0
+      - -e git+https://github.com/bird-house/twitcher@master#egg=pyramid_twitcher
       - pyramid-rpc
       - colander
       - Authomatic==0.1.0.post1
-      - python-openid==2.2.5
+      - python-openid2==3.0
       - WebHelpers2
       - WebHelpers2_grid
       - repoze.sendmail

phoenix/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.10.dev'
+__version__ = '0.10.dev0'
 
 
 def main(global_config, **settings):

phoenix/account/base.py

@@ -43,9 +43,9 @@ def generate_form(self):
 
     def process_form(self, form):
         try:
-            controls = self.request.POST.items()
+            controls = list(self.request.POST.items())
             appstruct = form.validate(controls)
-        except ValidationFailure, e:
+        except ValidationFailure as e:
             self.session.flash("<strong>Error:</strong> Login failed.", queue='danger')
             return dict(form=e.render())
         else:

phoenix/account/ldap.py

@@ -63,7 +63,7 @@ def init_ldap(self):
             ldap_settings = self.request.db.ldap.find_one()
 
             # Set LDAP settings
-            import ldap
+            from . import ldap
             if ldap_settings['scope'] == 'ONELEVEL':
                 ldap_scope = ldap.SCOPE_ONELEVEL
             else:

phoenix/catalog.py

@@ -126,7 +126,7 @@ def doc2record(document):
         if '_id' in document:
             # _id field not allowed in record
             del document["_id"]
-        record = namedtuple('Record', document.keys())(*document.values())
+        record = namedtuple('Record', list(document.keys()))(*list(document.values()))
     return record
 
 
@@ -147,7 +147,7 @@ def delete_record(self, identifier):
         self.collection.delete_one({'identifier': identifier})
 
     def insert_record(self, record):
-        record['identifier'] = uuid.uuid4().get_urn()
+        record['identifier'] = uuid.uuid4().hex
         self.collection.update_one({'source': record['source']}, {'$set': record}, True)
 
     def harvest(self, url, service_type, service_name=None, service_title=None, public=False, c4i=False):

phoenix/esgf/logon.py

@@ -3,7 +3,6 @@
 import tempfile
 import requests
 import re
-import six
 from lxml import etree
 from io import BytesIO
 import OpenSSL
@@ -58,9 +57,6 @@ def logon(username=None, password=None, hostname=None, interactive=False, outdir
     outdir = outdir or os.curdir
     lm = LogonManager(esgf_dir=outdir, dap_config=os.path.join(outdir, 'dodsrc'))
     lm.logoff()
-    # TODO: fix encoding
-    if six.PY2:
-        hostname = hostname.encode('utf-8', 'ignore')
     # logon
     lm.logon(username=username, password=password, hostname=hostname,
              bootstrap=True, update_trustroots=False, interactive=interactive)

phoenix/esgf/metadata.py

@@ -1,7 +1,7 @@
 from owslib.wps import WebProcessingService
 from pyesgf.multidict import MultiDict
 
-from phoenix._compat import urlparse
+from urllib.parse import urlparse
 
 import logging
 LOGGER = logging.getLogger("PHOENIX")
@@ -30,5 +30,5 @@ def convert_constraints(url):
         key, value = qpart.split('=')
         for val in value.split(','):
             constraints.add(key.strip(), val.strip())
-    converted = ','.join(["{0[0]}:{0[1]}".format(c) for c in constraints.iteritems()])
+    converted = ','.join(["{0[0]}:{0[1]}".format(c) for c in constraints.items()])
     return converted

phoenix/esgf/search.py

@@ -35,7 +35,7 @@ def date_from_filename(filename):
 
 def variable_filter(constraints, variables):
     """return True if variable fulfills contraints"""
-    var_types = [u'variable', u'cf_standard_name', u'variable_long_name']
+    var_types = ['variable', 'cf_standard_name', 'variable_long_name']
 
     success = True
     # check different types of variables
@@ -199,7 +199,6 @@ def _run_search_items(self, dataset_id, search_type):
         ctx = ctx.constrain(dataset_id=dataset_id)
         items = []
         LOGGER.debug("hit_count: %s", ctx.hit_count)
-        print ctx.hit_count
         for result in ctx.search():
             if search_type == TYPE_FILE and not temporal_filter(result.filename, self._start, self._end):
                 LOGGER.debug("skipped by temporal filter")
@@ -249,7 +248,7 @@ def search_datasets(self):
         pinned_keywords = []
         for facet in ctx.facet_counts:
             if facet not in self._constraints and len(ctx.facet_counts[facet]) == 1:
-                pinned_keywords.append("{}:{}".format(facet, ctx.facet_counts[facet].keys()[0]))
+                pinned_keywords.append("{}:{}".format(facet, list(ctx.facet_counts[facet].keys())[0]))
         pinned_keywords = sorted(pinned_keywords)
         projects = Counter(ctx.facet_counts['project']).most_common(7)
         paged_results = []

phoenix/esgf/slcsclient.py

@@ -1,6 +1,6 @@
 from OpenSSL import crypto
 from base64 import b64encode
-from StringIO import StringIO
+from io import StringIO
 
 from requests_oauthlib import OAuth2Session
 

phoenix/esgf/views/esgflogon.py

@@ -35,17 +35,17 @@ def generate_form(self):
 
     def process_form(self, form):
         try:
-            controls = self.request.POST.items()
+            controls = list(self.request.POST.items())
             appstruct = form.validate(controls)
             result = esgf_logon.delay(authenticated_userid(self.request),
                                       appstruct.get('provider'),
                                       appstruct.get('username'),
                                       appstruct.get('password'))
             self.session['task_id'] = result.id
-        except ValidationFailure, e:
+        except ValidationFailure as e:
             self.session.flash("Form validation failed.", queue='danger')
             return dict(form=e.render())
-        except Exception, e:
+        except Exception as e:
             msg = '<strong>Error:</strong> ESGF logon failed: {0}.'.format(escape(e.message))
             self.session.flash(msg, queue='danger')
             return HTTPFound(location=self.request.route_path('esgflogon'))

phoenix/events.py

@@ -19,6 +19,6 @@ def __init__(self, request, new_settings):
 
     def converted_settings(self):
         converted = {}
-        for key, value in self.new_settings.iteritems():
+        for key, value in self.new_settings.items():
             converted[key.replace('_', '.')] = value
         return converted

phoenix/geoform/form.py

@@ -1,5 +1,4 @@
 import os
-from UserDict import DictMixin
 
 import colander
 from pyramid.security import authenticated_userid

phoenix/monitor/panels/outputs.py

@@ -33,7 +33,7 @@ def __init__(self, context, request):
     def panel(self):
         job_id = self.request.matchdict.get('job_id')
         items = []
-        for output in process_outputs(self.request, job_id).values():
+        for output in list(process_outputs(self.request, job_id).values()):
             items.append(output_details(self.request, output))
         items = sorted(items, key=lambda item: item['identifier'], reverse=1)
         return dict(items=items)

phoenix/monitor/utils.py

@@ -7,7 +7,7 @@
 def escape_output(output):
     if output:
         if isinstance(output, list):
-            return map(escape, output)
+            return list(map(escape, output))
         else:
             return escape(output)
     else:

phoenix/monitor/views/actions.py

@@ -56,7 +56,7 @@ def delete_jobs(self):
         ids = self._selected_children()
         if ids is not None:
             self.collection.delete_many({'identifier': {'$in': ids}})
-            self.session.flash(u"Selected jobs were deleted.", queue='info')
+            self.session.flash("Selected jobs were deleted.", queue='info')
         return HTTPFound(location=self.request.route_path('monitor'))
 
     # @view_config(route_name='delete_all_jobs', permission='admin')
@@ -74,7 +74,7 @@ def make_public(self):
         ids = self._selected_children()
         if ids is not None:
             self.collection.update_many({'identifier': {'$in': ids}}, {'$addToSet': {'tags': 'public'}})
-            self.session.flash(u"Selected jobs were made public.", 'info')
+            self.session.flash("Selected jobs were made public.", 'info')
         return HTTPFound(location=self.request.route_path('monitor'))
 
     @view_config(route_name='make_private')
@@ -85,7 +85,7 @@ def make_private(self):
         ids = self._selected_children()
         if ids is not None:
             self.collection.update_many({'identifier': {'$in': ids}}, {'$pull': {'tags': 'public'}})
-            self.session.flash(u"Selected jobs were made private.", 'info')
+            self.session.flash("Selected jobs were made private.", 'info')
         return HTTPFound(location=self.request.route_path('monitor'))
 
     @view_config(route_name='set_favorite')
@@ -96,7 +96,7 @@ def set_favorite(self):
         ids = self._selected_children()
         if ids is not None:
             self.collection.update_many({'identifier': {'$in': ids}}, {'$addToSet': {'tags': 'fav'}})
-            self.session.flash(u"Set as favorite done.", 'info')
+            self.session.flash("Set as favorite done.", 'info')
         return HTTPFound(location=self.request.route_path('monitor'))
 
     @view_config(route_name='unset_favorite')
@@ -107,7 +107,7 @@ def unset_favorite(self):
         ids = self._selected_children()
         if ids is not None:
             self.collection.update_many({'identifier': {'$in': ids}}, {'$pull': {'tags': 'fav'}})
-            self.session.flash(u"Unset as favorite done.", 'info')
+            self.session.flash("Unset as favorite done.", 'info')
         return HTTPFound(location=self.request.route_path('monitor'))
 
     @view_config(renderer='json', name='edit_job.json')
@@ -138,20 +138,20 @@ def monitor_buttons(context, request):
     # if request.has_permission('admin'):
     #    buttons.append(ActionButton('delete_all_jobs', title=u'Delete all',
     #                                css_class=u'btn btn-danger'))
-    buttons.append(ActionButton('delete_jobs', title=u'Delete',
-                                css_class=u'btn btn-danger',
+    buttons.append(ActionButton('delete_jobs', title='Delete',
+                                css_class='btn btn-danger',
                                 disabled=not request.has_permission('edit')))
-    buttons.append(ActionButton('make_public', title=u'Make Public',
-                                css_class=u'btn btn-warning',
+    buttons.append(ActionButton('make_public', title='Make Public',
+                                css_class='btn btn-warning',
                                 disabled=not request.has_permission('edit')))
-    buttons.append(ActionButton('make_private', title=u'Make Private',
-                                css_class=u'btn btn-warning',
+    buttons.append(ActionButton('make_private', title='Make Private',
+                                css_class='btn btn-warning',
                                 disabled=not request.has_permission('edit')))
-    buttons.append(ActionButton('set_favorite', title=u'Set Favorite',
-                                css_class=u'btn btn-success',
+    buttons.append(ActionButton('set_favorite', title='Set Favorite',
+                                css_class='btn btn-success',
                                 disabled=not request.has_permission('edit')))
-    buttons.append(ActionButton('unset_favorite', title=u'Unset Favorite',
-                                css_class=u'btn btn-success',
+    buttons.append(ActionButton('unset_favorite', title='Unset Favorite',
+                                css_class='btn btn-success',
                                 disabled=not request.has_permission('edit')))
     return buttons
 

phoenix/monitor/views/list.py

@@ -89,7 +89,7 @@ def generate_caption_form(self, formid="deform_caption"):
 
     def process_caption_form(self, form):
         try:
-            controls = self.request.POST.items()
+            controls = list(self.request.POST.items())
             LOGGER.debug("controls %s", controls)
             appstruct = form.validate(controls)
             self.collection.update_one({'identifier': appstruct['identifier']},
@@ -113,7 +113,7 @@ def generate_labels_form(self, formid="deform_tags"):
 
     def process_labels_form(self, form):
         try:
-            controls = self.request.POST.items()
+            controls = list(self.request.POST.items())
             LOGGER.debug("controls %s", controls)
             appstruct = form.validate(controls)
             tags = make_tags(appstruct['labels'])
@@ -211,10 +211,10 @@ def labels_td(self, col_num, i, item):
     def buttongroup_td(self, col_num, i, item):
         from phoenix.utils import ActionButton
         buttons = list()
-        buttons.append(ActionButton('results', title=u'Details', css_class=u'btn btn-default',
+        buttons.append(ActionButton('results', title='Details', css_class='btn btn-default',
                                     href=self.request.route_path('job_details', tab='log',
                                                                  job_id=item.get('identifier'))))
-        buttons.append(ActionButton('restart_job', title=u'Restart', css_class=u'btn btn-default',
+        buttons.append(ActionButton('restart_job', title='Restart', css_class='btn btn-default',
                                     href="/restart_job/%s" % item.get('identifier'),
                                     disabled=item['status'] != 'ProcessSucceeded'))
         return self.render_buttongroup_td(buttons=buttons)