[Feature] Progress tracking and scheduling

.gitignore

@@ -6,12 +6,12 @@
 !.isort.cfg
 !setup.cfg
 *.orig
-*.log
 *.pot
 __pycache__/*
 .cache/*
 .*.swp
-*/.ipynb_checkpoints/*
+.ipynb_checkpoints/*
+*.sif
 
 # Project files
 .ropeproject
@@ -20,12 +20,16 @@ __pycache__/*
 .settings
 .idea
 tags
+.vscode
+
+output
 
 # Package files
 *.egg
 *.eggs/
 .installed.cfg
 *.egg-info
+pip-wheel-metadata
 
 # Unittest and coverage
 htmlcov/*

.travis.yml

@@ -49,7 +49,7 @@ install:
 before_script:
   - git config --global user.email "support@travis-ci.org"
   - git config --global user.name "Travis"
-  - singularity pull shub://FCP-INDI/C-PAC || singularity pull docker://fcpindi/c-pac:latest
+  # - singularity pull shub://FCP-INDI/C-PAC || singularity pull docker://fcpindi/c-pac:latest
 script:
   - coverage run --append -m pytest
   - coverage report -m

cpac-api.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=API to allow CPAC Dashboard to communicate with CPAC CLI (e.g. for cluster)
+After=sshd.service
+
+[Service]
+Type=simple
+User=ubuntu
+ExecStart=/home/ubuntu/miniconda3/bin/python -m cpac.api scheduler --backend slurm \
+  --singularity-image /shared/C-PAC.simg --slurm-host localhost:22 --slurm-username ubuntu
+WorkingDirectory=/home/ubuntu
+
+[Install]
+Alias=cpac-api.service
+WantedBy=mult-user.target

pytest.ini

@@ -0,0 +1,13 @@
+[pytest]
+log_cli = 1
+log_cli_level = INFO
+log_cli_format = %(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)
+log_cli_date_format=%Y-%m-%d %H:%M:%S
+addopts = -s -v
+
+testpaths = tests
+norecursedirs =
+    dist
+    build
+    .tox
+    tests/helpers

requirements-dev.txt

@@ -2,4 +2,8 @@ coveralls
 pytest
 pytest-remotedata >= 0.3.2
 pytest-runner
+pytest-cov
+pytest-runner
+pytest-tornado
+pytest-asyncio
 sphinx

requirements.txt

@@ -7,4 +7,4 @@ setuptools
 spython >= 0.0.81
 tabulate >= 0.8.6
 tornado
-websocket-client
+appdirs == 1.4.4

setup.cfg

@@ -44,7 +44,8 @@ install_requires =
     pyyaml
     tabulate >= 0.8.6
     tornado
-    websocket-client
+    appdirs
+    dataclasses; python_version < "3.7"
 python_requires = >=3.6
 
 [options.packages.find]
@@ -58,28 +59,14 @@ testing =
     imagesize
     pytest
     pytest-cov
-    pytest-runne
+    pytest-runner
+    pytest-tornado
+    pytest-asyncio
     sphinx
 [options.entry_points]
 console_scripts =
     cpac = cpac.__main__:run
 
-[test]
-addopts = --verbose -s
-extras = True
-
-
-[tool:pytest]
-addopts = -s
-# --cov cpac --cov-report term-missing -s
-# -s
-# --verbose
-norecursedirs =
-    dist
-    build
-    .tox
-testpaths = tests
-
 [aliases]
 build = bdist_wheel
 release = build upload

src/cpac/__main__.py

@@ -15,19 +15,6 @@
 clargs = {'group', 'utils'}
 
 
-class ExtendAction(argparse.Action):
-    def __call__(self, parser, namespace, values, option_string=None):
-        items = (getattr(namespace, self.dest) or []) + values
-        items = [x for n, x in enumerate(items) if x not in items[:n]]
-        setattr(namespace, self.dest, items)
-
-
-def address(str):  # pragma: no cover
-    addr, port = str.split(':')
-    port = int(port)
-    return addr, port
-
-
 def parse_args(args):
     cwd = os.getcwd()
 
@@ -142,8 +129,6 @@ def parse_args(args):
     )
 
     help_call = '--help' in sys.argv or '-h' in sys.argv
-    run_parser.register('action', 'extend', ExtendAction)
-    # run_parser.add_argument('--address', action='store', type=address)
 
     if not help_call:
         # These positional arguments are required unless we're just getting
@@ -173,21 +158,18 @@ def parse_args(args):
         add_help=False,
         formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
-    group_parser.register('action', 'extend', ExtendAction)
 
     utils_parser = subparsers.add_parser(
         'utils',
         add_help=False,
         formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
-    utils_parser.register('action', 'extend', ExtendAction)
 
     crash_parser = subparsers.add_parser(
         'crash',
         add_help=True,
         formatter_class=argparse.ArgumentDefaultsHelpFormatter
     )
-    crash_parser.register('action', 'extend', ExtendAction)
 
     crash_parser.add_argument(
         'crashfile',

src/cpac/api/__main__.py

@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import argparse
+import sys
+import logging
+import random
+import string
+
+from .. import __version__
+
+_logger = logging.getLogger(__name__)
+
+
+class ExtendAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        items = (getattr(namespace, self.dest) or []) + values
+        items = [x for n, x in enumerate(items) if x not in items[:n]]
+        setattr(namespace, self.dest, items)
+
+
+def address(str):
+    addr, port = str.split(':')
+    port = int(port)
+    return addr, port
+
+
+def parse_args(args):
+    parser = argparse.ArgumentParser(
+        description='cpac: a Python package that simplifies using C-PAC. CCs development version'
+    )
+
+    parser.add_argument(
+        '--version',
+        action='version',
+        version='cpac {ver}'.format(ver=__version__)
+    )
+
+    parser.add_argument(
+        '-v',
+        '--verbose',
+        dest="loglevel",
+        help="set loglevel to INFO",
+        action='store_const',
+        const=logging.INFO
+    )
+
+    parser.add_argument(
+        '-vv',
+        '--very-verbose',
+        dest="loglevel",
+        help="set loglevel to DEBUG",
+        action='store_const',
+        const=logging.DEBUG
+    )
+
+    subparsers = parser.add_subparsers(dest='command')
+
+    scheduler_parser = subparsers.add_parser('scheduler')
+    scheduler_parser.add_argument('--address', action='store', type=address, default='localhost:3333')
+    scheduler_parser.add_argument('--proxy', action='store_true')
+    scheduler_parser.add_argument('--backend', choices=['docker', 'singularity', 'slurm'], default='singularity')
+
+    scheduler_parser.add_argument('--singularity-image', nargs='?')
+
+    scheduler_parser.add_argument('--docker-image', nargs='?')
+
+    scheduler_parser.add_argument('--slurm-host', nargs='?')
+    scheduler_parser.add_argument('--slurm-username', nargs='?')
+    scheduler_parser.add_argument('--slurm-key', nargs='?')
+    scheduler_parser.add_argument('--slurm-control', nargs='?',
+            default=f'~/.ssh/{"".join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(24))}')
+    scheduler_parser.add_argument('--slurm-pip-install', nargs='?')
+    scheduler_parser.add_argument('--slurm-singularity-image', nargs='?')
+
+    parsed = parser.parse_args(args)
+
+    return parsed
+
+
+def setup_logging(loglevel):
+    logformat = "[%(asctime)s] %(levelname)s:%(name)s:%(message)s"
+    logging.basicConfig(level=loglevel, stream=sys.stdout,
+                        format=logformat, datefmt="%Y-%m-%d %H:%M:%S")
+
+
+async def start(args):
+    from cpac.api.server import start
+    from cpac.api.backends import available_backends
+    from cpac.api.scheduling import Scheduler
+    from cpac.api.authKey import AuthKey
+
+    print("Running server")
+    print("Auth key: ", AuthKey.generate_key())
+
+    backend = args.backend
+    cmd_args = vars(args)
+
+    backend = available_backends[backend](
+        id=backend,
+        **{
+            arg.split('_', 1)[1]: val
+            for arg, val in cmd_args.items()
+            if arg.startswith(backend)
+        }
+    )
+
+    async with Scheduler(backend, proxy=args.proxy) as scheduler:
+        await start(args.address, scheduler)
+        await scheduler
+
+
+def main(args):
+    command = args[0]
+    args = parse_args(args[1:])
+    setup_logging(args.loglevel)
+
+    if args.command == 'scheduler':
+        import asyncio
+        asyncio.run(start(args))
+
+
+def run():
+    main(sys.argv)
+
+
+if __name__ == "__main__":
+    run()

src/cpac/api/authKey.py

@@ -0,0 +1,15 @@
+import uuid
+
+
+class AuthKey:
+    key = None
+
+    @staticmethod
+    def generate_key():
+        if not AuthKey.key:
+            AuthKey.key = str(uuid.uuid1())
+        return AuthKey.key
+
+    @staticmethod
+    def get_key():
+        return AuthKey.key

src/cpac/api/backends/__init__.py

@@ -0,0 +1,9 @@
+from .docker import DockerBackend
+from .singularity import SingularityBackend
+from .slurm import SLURMBackend
+
+available_backends = {
+    'docker': DockerBackend,
+    'singularity': SingularityBackend,
+    'slurm': SLURMBackend,
+}