-
Notifications
You must be signed in to change notification settings - Fork 395
/
downloads.py
148 lines (110 loc) · 4.81 KB
/
downloads.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
import json
import logging
import os
import re
import subprocess
import sys
import tempfile
import zipfile
# from distutils.version import StrictVersion
from packaging.version import parse as StrictVersion
import jsonschema
import requests
base_dir = os.path.dirname(os.path.abspath(__file__))
repo_dir = os.path.join(base_dir, "site", "dat", "repo")
dest_dir = os.path.join(base_dir, "site", "files", "packages")
def is_version_packed(fname):
resp = requests.head("https://jmeter-plugins.org/files/packages/%s" % fname)
resp.close()
return resp.status_code == 200
def get_packages():
resp = requests.get("http://jmeter-plugins.org/files/packages/")
assert resp.status_code == 200
return {x['file']: x['size'] for x in json.loads(resp.content)['files']}
def pack_version(fname, ver_obj, pmgr_obj, installer_cls):
if not ver_obj['downloadUrl']:
return
logging.info("Packing into %s", fname)
with zipfile.ZipFile(fname, 'w', zipfile.ZIP_DEFLATED) as ziph:
tmp_dir = tempfile.mkdtemp()
# download main jar
jar_path = download_into_dir(tmp_dir, ver_obj['downloadUrl'], os.path.join("lib", "ext"))
if installer_cls is not None:
os.mkdir(os.path.join(tmp_dir, 'bin'))
subprocess.check_call(["java", "-cp", jar_path, installer_cls])
# download libs
if 'libs' in ver_obj:
for libname in ver_obj['libs']:
download_into_dir(tmp_dir, ver_obj['libs'][libname], os.path.join("lib"))
# download pmgr
download_into_dir(tmp_dir, pmgr_obj['downloadUrl'], os.path.join("lib", "ext"))
# archive temp folder
zip_dir(tmp_dir, ziph)
def zip_dir(path, ziph):
# ziph is zipfile handle
for root, dirs, files in os.walk(path):
for fname in files:
ziph.write(os.path.join(root, fname), os.path.join(root[len(path):], fname))
def download_into_dir(dirname, url, dest_subpath):
logging.info("Downloading: %s", url)
logging.debug("Dest dir: %s / subpath: %s", dirname, dest_subpath)
resp = requests.get(url)
assert resp.status_code == 200
if 'content-disposition' in resp.headers:
# handle optional quotes around filename:
remote_filename = re.sub('.*filename=(")?(?P<filename>.+)(?(1)"|$).*', "\g<filename>",
resp.headers['content-disposition'])
else:
remote_filename = os.path.basename(resp.url)
dir_path = os.path.join(dirname, dest_subpath)
if not os.path.exists(dir_path):
os.makedirs(dir_path)
with open(os.path.join(dirname, dest_subpath, remote_filename), 'wb') as fname:
fname.write(resp.content)
fname.close()
resp.close()
return os.path.join(dir_path, remote_filename)
def get_pmgr(plugins_list):
for plug in plugins_list:
if plug['id'] == 'jpgc-plugins-manager':
versions = sorted(plug['versions'].keys(), key=StrictVersion)
return plug['versions'][versions[-1]]
raise Exception("Failed to find plugins manager meta info")
if __name__ == "__main__":
level = logging.DEBUG if os.getenv("DEBUG") else logging.INFO
logging.basicConfig(level=level, format="%(relativeCreated)d\t%(levelname)s\t%(message)s")
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
with open("repo_schema.json") as fhd:
schema = json.load(fhd)
plugins = []
for repo_file in os.listdir(repo_dir):
with open(os.path.join(repo_dir, repo_file)) as fhd:
content = json.loads(fhd.read())
jsonschema.validate(content, schema)
plugins.extend(content)
if len(sys.argv) > 1:
logging.info("Doing no downloads, just checked JSON validity")
sys.exit(0)
known_packages = get_packages()
# find pmgr
pmgr_obj = get_pmgr(plugins)
download_into_dir(tempfile.mkdtemp(), pmgr_obj['downloadUrl'], "pmgr") # TODO: use it as cached
for n, plugin in enumerate(plugins):
if 'screenshotUrl' not in plugin:
raise ValueError("%s has no screenshotUrl" % plugin['id'])
logging.debug("Processing plugin %d of %d: %s", n, len(plugins), plugin['id'])
if plugin['id'] == 'jpgc-plugins-manager':
continue
for version in plugin['versions']:
if not version:
continue
logging.debug("Version: %s", version)
if not plugin['versions'][version]['downloadUrl']:
continue
dest_file = "%s-%s.zip" % (plugin['id'], version)
if known_packages.get(dest_file) or is_version_packed(dest_file):
logging.info("Skip: %s %s", plugin['id'], version)
continue
pack_version(os.path.join(dest_dir, dest_file), plugin['versions'][version], pmgr_obj,
plugin.get('installerClass'))