diff options
author | Steve Dower <steve.dower@microsoft.com> | 2017-07-19 09:11:08 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-07-19 09:11:08 +0200 |
commit | e99d3a52a50b3f836fb9fb88f317aacddd494858 (patch) | |
tree | 3347183e2ddbd4948ac3070d96505ca6a38908f8 /PCbuild | |
parent | 241c4a28d9fec7f2beffc9cbd21f156999959a9d (diff) | |
download | cpython-git-e99d3a52a50b3f836fb9fb88f317aacddd494858.tar.gz |
[3.6] bpo-30450: Improved logic for obtaining dependencies (#2751)
Adds alternate download approach for nuget.exe
Fall back to git.exe if no Python is found. (#2739)
Also check whether git.exe is on PATH if it will be used.
Add support for HOST_PYTHON variable.
Clear internal environment variables used in find_python.bat
Use HOST_PYTHON as the actual Python if it is recent enough.
Adds HOST_PYTHON variable to AppVeyor configuration
Diffstat (limited to 'PCbuild')
-rw-r--r-- | PCbuild/build.bat | 2 | ||||
-rw-r--r-- | PCbuild/find_python.bat | 75 | ||||
-rw-r--r-- | PCbuild/get_externals.bat | 34 | ||||
-rw-r--r-- | PCbuild/urlretrieve.py | 39 |
4 files changed, 127 insertions, 23 deletions
diff --git a/PCbuild/build.bat b/PCbuild/build.bat index b63843fa95..81e500d554 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -5,8 +5,6 @@ echo.%~nx0 [flags and arguments] [quoted MSBuild options] echo. echo.Build CPython from the command line. Requires the appropriate echo.version(s) of Microsoft Visual Studio to be installed (see readme.txt). -echo.Also requires Subversion (svn.exe) to be on PATH if the '-e' flag is -echo.given. echo. echo.After the flags recognized by this script, up to 9 arguments to be passed echo.directly to MSBuild may be passed. If the argument contains an '=', the diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat new file mode 100644 index 0000000000..ba3a0f5a10 --- /dev/null +++ b/PCbuild/find_python.bat @@ -0,0 +1,75 @@ +@rem +@rem Searches for python.exe and may download a private copy from nuget. +@rem +@rem This file is supposed to modify the state of the caller (specifically +@rem the MSBUILD variable), so we do not use setlocal or echo, and avoid +@rem changing any other persistent state. +@rem + +@rem No arguments provided means do full search +@if '%1' EQU '' goto :begin_search + +@rem One argument may be the full path. Use a goto so we don't try to +@rem parse the next if statement - incorrect quoting in the multi-arg +@rem case can cause us to break immediately. +@if '%2' EQU '' goto :one_arg + +@rem Entire command line may represent the full path if quoting failed. +@if exist "%*" (set PYTHON="%*") & (set _Py_Python_Source=from environment) & goto :found +@goto :begin_search + +:one_arg +@if exist "%~1" (set PYTHON="%~1") & (set _Py_Python_Source=from environment) & goto :found + +:begin_search +@set PYTHON= + +@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR% +@if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals) + +@rem If we have Python in externals, use that one +@if exist "%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe" (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found in externals directory) & goto :found + +@rem If HOST_PYTHON is recent enough, use that +@if NOT "%HOST_PYTHON%"=="" @%HOST_PYTHON% -c "import sys; assert sys.version_info[:2] >= (3, 6)" >nul 2>nul && (set PYTHON="%HOST_PYTHON%") && (set _Py_Python_Source=found as HOST_PYTHON) && goto :found + +@rem If py.exe finds a recent enough version, use that one +@py -3.6 -V >nul 2>&1 && (set PYTHON=py -3.6) && (set _Py_Python_Source=found with py.exe) && goto :found + +@if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" +@set _Py_NUGET=%NUGET% +@set _Py_NUGET_URL=%NUGET_URL% +@set _Py_HOST_PYTHON=%HOST_PYTHON% +@if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py +@if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe) +@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl) +@if NOT exist "%_Py_NUGET%" ( + @echo Downloading nuget... + @rem NB: Must use single quotes around NUGET here, NOT double! + @rem Otherwise, a space in the path would break things + @rem If it fails, retry with any available copy of Python + @powershell.exe -Command Invoke-WebRequest %_Py_NUGET_URL% -OutFile '%_Py_NUGET%' + @if errorlevel 1 ( + @%_Py_HOST_PYTHON% "%~dp0\urlretrieve.py" "%_Py_NUGET_URL%" "%_Py_NUGET%" + ) +) +@echo Installing Python via nuget... +@"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" +@rem Quote it here; it's not quoted later because "py -3.6" wouldn't work +@if not errorlevel 1 (set PYTHON="%_Py_EXTERNALS_DIR%\pythonx86\tools\python.exe") & (set _Py_Python_Source=found on nuget.org) & goto :found + + +@set _Py_Python_Source= +@set _Py_EXTERNALS_DIR= +@set _Py_NUGET= +@set _Py_NUGET_URL= +@set _Py_HOST_PYTHON= +@exit /b 1 + +:found +@echo Using %PYTHON% (%_Py_Python_Source%) +@set _Py_Python_Source= +@set _Py_EXTERNALS_DIR= +@set _Py_NUGET= +@set _Py_NUGET_URL= +@set _Py_HOST_PYTHON= diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index 6e466a3448..0a86892c35 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -4,8 +4,6 @@ rem Simple script to fetch source for external libraries if "%PCBUILD%"=="" (set PCBUILD=%~dp0) if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%PCBUILD%\..\externals) -if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\nuget.exe) -if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl) set DO_FETCH=true set DO_CLEAN=false @@ -13,7 +11,7 @@ set DO_CLEAN=false :CheckOpts if "%~1"=="--no-tkinter" (set IncludeTkinter=false) & shift & goto CheckOpts if "%~1"=="--no-openssl" (set IncludeSSL=false) & shift & goto CheckOpts -if "%~1"=="--python" (set PYTHON_FOR_BUILD=%2) & shift & shift & goto CheckOpts +if "%~1"=="--python" (set PYTHON=%2) & shift & shift & goto CheckOpts if "%~1"=="--organization" (set ORG=%2) & shift & shift & goto CheckOpts if "%~1"=="-c" (set DO_CLEAN=true) & shift & goto CheckOpts if "%~1"=="--clean" (set DO_CLEAN=true) & shift & goto CheckOpts @@ -33,22 +31,10 @@ if "%DO_FETCH%"=="false" goto end if "%ORG%"=="" (set ORG=python) -if "%PYTHON_FOR_BUILD%"=="" ( - echo Checking for installed python... - py -3.6 -V >nul 2>&1 && (set PYTHON_FOR_BUILD=py -3.6) -) -if "%PYTHON_FOR_BUILD%"=="" ( - if NOT exist "%EXTERNALS_DIR%" mkdir "%EXTERNALS_DIR%" - if NOT exist "%NUGET%" ( - echo Downloading nuget... - rem NB: Must use single quotes around NUGET here, NOT double! - rem Otherwise, a space in the path would break things - powershell.exe -Command Invoke-WebRequest %NUGET_URL% -OutFile '%NUGET%' - ) - echo Installing Python via nuget... - "%NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%EXTERNALS_DIR%" - rem Quote it here; it's not quoted later because "py -3.6" wouldn't work - set PYTHON_FOR_BUILD="%EXTERNALS_DIR%\pythonx86\tools\python.exe" +call "%PCBUILD%find_python.bat" "%PYTHON%" + +if "%PYTHON%"=="" ( + where /Q git || echo Python 3.6 could not be found or installed, and git.exe is not on your PATH && exit /B 1 ) echo.Fetching external libraries... @@ -65,9 +51,12 @@ set libraries=%libraries% xz-5.2.2 for %%e in (%libraries%) do ( if exist "%EXTERNALS_DIR%\%%e" ( echo.%%e already exists, skipping. + ) else if "%PYTHON%"=="" ( + echo.Fetching %%e with git... + git clone --depth 1 https://github.com/%ORG%/cpython-source-deps --branch %%e "%EXTERNALS_DIR%\%%e" ) else ( echo.Fetching %%e... - %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -O %ORG% %%e + %PYTHON% "%PCBUILD%get_external.py" -O %ORG% %%e ) ) @@ -80,9 +69,12 @@ if NOT "%IncludeSSL%"=="false" set binaries=%binaries% nasm-2.11.06 for %%b in (%binaries%) do ( if exist "%EXTERNALS_DIR%\%%b" ( echo.%%b already exists, skipping. + ) else if "%PYTHON%"=="" ( + echo.Fetching %%b with git... + git clone --depth 1 https://github.com/%ORG%/cpython-bin-deps --branch %%b "%EXTERNALS_DIR%\%%b" ) else ( echo.Fetching %%b... - %PYTHON_FOR_BUILD% "%PCBUILD%get_external.py" -b -O %ORG% %%b + %PYTHON% "%PCBUILD%get_external.py" -b -O %ORG% %%b ) ) diff --git a/PCbuild/urlretrieve.py b/PCbuild/urlretrieve.py new file mode 100644 index 0000000000..9df773c35d --- /dev/null +++ b/PCbuild/urlretrieve.py @@ -0,0 +1,39 @@ +# Simple Python script to download a file. Used as a fallback +# when other more reliable methods fail. +# +from __future__ import print_function +import sys + +try: + from requests import get +except ImportError: + try: + from urllib.request import urlretrieve + USING = "urllib.request.urlretrieve" + except ImportError: + try: + from urllib import urlretrieve + USING = "urllib.retrieve" + except ImportError: + print("Python at", sys.executable, "is not suitable", + "for downloading files.", file=sys.stderr) + sys.exit(2) +else: + USING = "requests.get" + + def urlretrieve(url, filename): + r = get(url, stream=True) + r.raise_for_status() + with open(filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=1024): + f.write(chunk) + return filename + +if __name__ == '__main__': + if len(sys.argv) != 3: + print("Usage: urlretrieve.py [url] [filename]", file=sys.stderr) + sys.exit(1) + URL = sys.argv[1] + FILENAME = sys.argv[2] + print("Downloading from", URL, "to", FILENAME, "using", USING) + urlretrieve(URL, FILENAME) |