[WIP] everware.yml with volumes #190 #193
Conversation
mount files downloaded from <url> into /notebook/data can't mount in /notebooks (it deletes git files)
everware/git_processor.py
Outdated
| text = yaml.load(file) | ||
| volumes = [] | ||
| for element in text: | ||
| volumes += text.get(element).get("volumes") |
There was a problem hiding this comment.
let's rename it into 'data' ('volumes' is already taken)
also it should be possible to specify exact location inside container where data volume should be mapped (or use '/notebook/data' by default)
everware/git_processor.py
Outdated
| self.download_file(volume) | ||
|
|
||
| def download_file(self, url): | ||
| import urllib |
There was a problem hiding this comment.
it should put some info into log about starting download, file size, etc
everware/spawner.py
Outdated
| @@ -371,6 +373,9 @@ def start(self, image=None): | |||
| yield self.remove_old_container() | |||
| self.log.info("Starting container from image: %s" % image_name) | |||
| self._add_to_log('Creating container') | |||
| #can't mount into /notebooks folder - it deletes all git files | |||
| if self.everware_yml_param.get("directory_volume"): | |||
| self.volumes = {self.everware_yml_param.get("directory_volume") : "/notebook/data"} | |||
There was a problem hiding this comment.
directory to map data folder to ('/notebook/data') should be defined inside parse_everware_yml
everware/git_processor.py
Outdated
| @@ -177,3 +184,30 @@ def load_state(self, state): | |||
| if key in state: | |||
| setattr(self, key, state[key]) | |||
|
|
|||
| def parse(self, everware_yml): | |||
There was a problem hiding this comment.
make function name more specific, parse_everware_yml?
everware/git_processor.py
Outdated
| directory_volume = self._repo_dir + "-volume" | ||
| os.mkdir(directory_volume) | ||
| try: | ||
| urllib.request.urlretrieve(url, directory_volume + '/' + filename) |
There was a problem hiding this comment.
I think this call is blocking, so the server will stop being responsive while doing this. Take a look at the OAuthAuthenticator and co for an example of using a non-blocking HTTP client in tornado
Conflicts: everware/git_processor.py
xrdcp doesn't die with timeout if connection doesn't exist \
| @@ -1,9 +1,13 @@ | |||
| import re | |||
| import git | |||
| from concurrent.futures import ThreadPoolExecutor | |||
| from tornado import gen | |||
| from tornado import gen,concurrent | |||
| setattr(self, key, state[key]) | ||
|
|
||
| def download_file(self, url): |
| try: | ||
| n_check_connection = 1 | ||
| self.log.info("Downloading from xrootd server %s" % url_struct.hostname) |
| http_client.close() | ||
| except: | ||
| self.log.info("Something went wrong during downloading from %s " % url_struct.hostname) |
| @@ -118,6 +122,34 @@ def prepare_local_repo(self): | |||
| else: | |||
| return True | |||
|
|
|||
| @gen.coroutine | |||
| def prepare_everware_yml(self, _user_log): | |||
There was a problem hiding this comment.
I'd call it 'parse_repo_everware_yml'
| self.log.info("Downloading from http server %s" % url_struct.hostname) | ||
| response = yield http_client.fetch(url_struct.geturl()) | ||
| with open(self.directory_data + '/' + filename, "ab") as f: |
| for data in all_data: | ||
| if type(data) == dict: | ||
| url = data.get("url") | ||
| ssl = data.get("ssl") |
There was a problem hiding this comment.
ssl variable should be initialised above
mount files downloaded from into /notebook/data
can't mount in /notebooks (it deletes git files)