Be more agressive trying to install requirements.
When -r requirements.txt is used, either then entire file works
or the entire file doesn't. This means that if there is a hiccup
with any of the files, we lose the entire run instead of just
the one file.
By iterating over the list and installing as many as we can, then
processing the freeze, we get as many of them as we can.
This is especially helpful when combined with the upcoming pbr/jeepyb
gate, where the lack of a requirement in the gate will show up when
the thing that needs it can't install it. If we get everything else,
the error message in that case will be much more clear as to why.
Change-Id: I27606b92a8be2605fa30362aaab51e65a21770bc
diff --git a/jeepyb/cmd/run_mirror.py b/jeepyb/cmd/run_mirror.py
index 8e84479..13bdd83 100644
--- a/jeepyb/cmd/run_mirror.py
+++ b/jeepyb/cmd/run_mirror.py
@@ -158,7 +158,7 @@
def build_mirror(self, mirror):
print("Building mirror: %s" % mirror['name'])
pip_format = ("%s install -M -U %s --exists-action=w "
- "--download-cache=%s --build %s -r %s")
+ "--download-cache=%s --build %s %s")
venv_format = ("virtualenv --clear --extra-search-dir=%s %s")
workdir = tempfile.mkdtemp()
@@ -219,41 +219,39 @@
new_reqs = self.process_http_requirements(reqlist,
pip_cache_dir,
pip)
- (reqfp, reqfn) = tempfile.mkstemp()
- os.write(reqfp, '\n'.join(new_reqs))
- os.close(reqfp)
+
+ for req in new_reqs:
+ out = self.run_command(pip_format %
+ (pip, "", pip_cache_dir,
+ build, req))
+ if "\nSuccessfully installed " not in out:
+ sys.stderr.write(
+ "Installing pip requires for %s:%s "
+ "failed.\n%s\n" % (project, branch, out))
+ print("pip install did not indicate success")
+ freeze = self.run_command("%s freeze -l" % pip)
+ requires = self.find_pkg_info(build)
+ reqfd = open(reqs, "w")
+ for line in freeze.split("\n"):
+ if line.startswith("-e ") or (
+ "==" in line and " " not in line):
+ requires.add(line)
+ for r in requires:
+ reqfd.write(r + "\n")
+ reqfd.close()
+ out = self.run_command(venv_format %
+ (pip_cache_dir, venv))
+ if os.path.exists(build):
+ shutil.rmtree(build)
out = self.run_command(pip_format %
- (pip, "", pip_cache_dir,
- build, reqfn))
- if "\nSuccessfully installed " not in out:
- sys.stderr.write("Installing pip requires for %s:%s "
- "failed.\n%s\n" %
+ (pip, "--no-install",
+ pip_cache_dir, build, reqs))
+ if "\nSuccessfully downloaded " not in out:
+ sys.stderr.write("Downloading pip requires for "
+ "%s:%s failed.\n%s\n" %
(project, branch, out))
print("pip install did not indicate success")
- else:
- freeze = self.run_command("%s freeze -l" % pip)
- requires = self.find_pkg_info(build)
- reqfd = open(reqs, "w")
- for line in freeze.split("\n"):
- if line.startswith("-e ") or (
- "==" in line and " " not in line):
- requires.add(line)
- for r in requires:
- reqfd.write(r + "\n")
- reqfd.close()
- out = self.run_command(venv_format %
- (pip_cache_dir, venv))
- if os.path.exists(build):
- shutil.rmtree(build)
- out = self.run_command(pip_format %
- (pip, "--no-install",
- pip_cache_dir, build, reqs))
- if "\nSuccessfully downloaded " not in out:
- sys.stderr.write("Downloading pip requires for "
- "%s:%s failed.\n%s\n" %
- (project, branch, out))
- print("pip install did not indicate success")
- print("cached:\n%s" % freeze)
+ print("cached:\n%s" % freeze)
else:
print("no requirements")
shutil.rmtree(workdir)