Revision fadc5cf0
Added by Benoît PECCATTE over 7 years ago
rudder-server-relay/SOURCES/relay-api/.gitignore | ||
---|---|---|
flask
|
||
*.pyc
|
rudder-server-relay/SOURCES/relay-api/README | ||
---|---|---|
How to setup the API
|
||
--------------------
|
||
|
||
Initialize the virtualenv with
|
||
$ ./virtualenv.py flask
|
||
|
||
Activate the virtualenv
|
||
$ . flask/bin/activate
|
||
|
||
Install dependencies
|
||
$ pip install -r requirements.txt
|
||
|
||
Modify and include apache/relay-api.conf in your apache configuration
|
||
|
||
How to test the shared-files API
|
||
--------------------------------
|
||
|
||
Have a nodeslist.json in /var/rudder/cfengine-community/inputs/distributePolicy/1.0/nodeslist.json
|
||
Have a policy_server in /var/rudder/cfengine-community/policy_server.dat
|
||
Have a uuid file in /opt/rudder/etc/uuid.hive
|
||
Hase a key pair in /var/rudder/cfengine-community/ppkeys/localhost.{priv,pub}
|
||
|
||
Launch a local API server
|
||
$ ./run.py
|
||
|
||
Create a signature (you need a cfengine key)
|
||
$ /opt/rudder/bin/rudder-sign file
|
||
|
||
Add the public key and TTL
|
||
$ echo pubkey=$(cat /var/rudder/cfengine-community/ppkeys/localhost.pub | grep -v -- -- | perl -pe 's/\n//g'i) >> file.sign
|
||
$ echo "ttl=1m" >> file.sign
|
||
|
||
Create the data to send
|
||
$ echo "" | cat file.sign - file > putdata
|
||
|
||
Send the data file
|
||
$ curl -T putdata http://127.0.0.1:5000/shared-files/target_uuid/source_uuid/filename
|
||
|
||
Test a file presence
|
||
$ curl -w "%{http_code}\n" -X HEAD http://127.0.0.1:5000/shared-files/target_uuid/source_uuid/filename?hash=....
|
||
|
rudder-server-relay/SOURCES/relay-api/apache/relay-api.conf | ||
---|---|---|
# Set up a WSGI serving process
|
||
WSGIDaemonProcess relay_api threads=5
|
||
WSGISocketPrefix /var/run/wsgi
|
||
|
||
## Set directory access permissions
|
||
|
||
<Directory /opt/rudder/share/relay-api>
|
||
# Allow access from anybody
|
||
Allow from all
|
||
</Directory>
|
||
|
||
<Directory /opt/rudder/share/relay-api/apache>
|
||
|
||
# WSGI parameters
|
||
WSGIProcessGroup relay_api
|
||
WSGIApplicationGroup %{GLOBAL}
|
||
|
||
# Allow access from anybody
|
||
Allow from all
|
||
|
||
</Directory>
|
||
|
||
# TODO
|
||
# WSGIScriptAlias /relay-api /opt/rudder/share/relay-api/apache/relay-api.wsgi
|
rudder-server-relay/SOURCES/relay-api/apache/relay-api.wsgi | ||
---|---|---|
# Import core modules
|
||
import sys
|
||
|
||
# Set up paths
|
||
api_path = '/opt/rudder/share/relay-api'
|
||
virtualenv_path = '/opt/rudder/share/relay-api'
|
||
|
||
# Virtualenv initialization
|
||
activate_this = virtualenv_path + '/bin/activate_this.py'
|
||
execfile(activate_this, dict(__file__=activate_this))
|
||
|
||
# Append ncf API path to the current one
|
||
sys.path.append(api_path)
|
||
|
||
# Launch
|
||
from relay_api import app as application
|
rudder-server-relay/SOURCES/relay-api/cleanup.sh | ||
---|---|---|
#!/bin/sh
|
||
|
||
# remove all files and their metadata in BASEDIR that have expired
|
||
|
||
BASEDIR="/var/rudder/shared-files"
|
||
date=$(date +%s)
|
||
find "${BASEDIR}" -type f -name '*.metadata' | xargs grep 'expires=' | sed 's/^\(.*\).metadata:expires=/\1 /' |
|
||
while read f d
|
||
do
|
||
if [ ${date} -gt ${d} ]
|
||
then
|
||
echo rm "${f}" "${f}.metadata"
|
||
fi
|
||
done
|
rudder-server-relay/SOURCES/relay-api/nodeslist.json.example | ||
---|---|---|
{
|
||
"c5e38f75-3fbe-402f-ac78-4ada2ea784a5": {
|
||
"hostname": "host1.rudder.local",
|
||
"key-hash": "sha1:da39a3ee5e6b4b0d3255bfef95601890afd80709",
|
||
"policy-server": "9b25d384-a0ed-4ac1-adf2-b5b0277a455d"
|
||
},
|
||
"6b26bf62-fe19-45ba-a0bc-4a1ddeca52ef": {
|
||
"hostname": "host2.rudder.local",
|
||
"key-hash": "sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||
"policy-server": "9b25d384-a0ed-4ac1-adf2-b5b0277a455d"
|
||
},
|
||
"465b0ed4-6cad-4dc3-a0ec-cf84c8e1d752": {
|
||
"hostname": "host3.rudder.local",
|
||
"key-hash": "sha512:cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
|
||
"policy-server": "6b26bf62-fe19-45ba-a0bc-4a1ddeca52ef"
|
||
},
|
||
"bf745c11-e995-4b4c-806a-86604708cfc3": {
|
||
"hostname": "host4.rudder.local",
|
||
"key-hash": "sha1:0539a3ee5e6b4b0d3255bfef95601890afd80710",
|
||
"policy-server": "465b0ed4-6cad-4dc3-a0ec-cf84c8e1d752"
|
||
}
|
||
}
|
rudder-server-relay/SOURCES/relay-api/relay_api/__init__.py | ||
---|---|---|
from flask import Flask
|
||
|
||
app = Flask(__name__)
|
||
|
||
from relay_api import views
|
rudder-server-relay/SOURCES/relay-api/relay_api/common.py | ||
---|---|---|
import traceback
|
||
import json
|
||
from flask import Flask, jsonify, request, abort, make_response
|
||
|
||
NODES=None
|
||
|
||
|
||
# Format error as api output
|
||
def format_error(exception, debug):
|
||
message = "Internal error!\n"
|
||
message += "Cause: " + unicode(exception) + "\n"
|
||
if debug:
|
||
message += traceback.format_exc() + "\n"
|
||
error = make_response(message)
|
||
error.headers['Content-Type'] = 'text/plain; charset=utf-8'
|
||
error.status_code = 500
|
||
return error
|
||
|
||
|
||
# Format a success response as api output
|
||
def format_response(text, code=200):
|
||
response = make_response(text)
|
||
response.headers['Content-Type'] = 'text/plain; charset=utf-8'
|
||
response.status_code = code
|
||
return response
|
||
|
||
|
||
# returns the UUID of localhost
|
||
def get_file_content(filename):
|
||
fd = open(filename, 'r')
|
||
content = fd.read().replace('\n','').strip()
|
||
fd.close()
|
||
return content
|
||
|
||
|
||
# returns [ relay_uuid, ... , node_uuid ], not including self
|
||
def node_route(nodes, my_uuid, uuid):
|
||
if uuid not in nodes:
|
||
raise ValueError("ERROR unknown node: " + str(uuid))
|
||
if "policy-server" not in nodes[uuid]:
|
||
raise ValueError("ERROR invalid nodes file on the server for " + uuid)
|
||
server = nodes[uuid]["policy-server"]
|
||
if server == my_uuid:
|
||
return [uuid]
|
||
route = node_route(nodes, my_uuid, server)
|
||
route.append(uuid)
|
||
return route
|
||
|
||
|
||
# Returns the parsed content of the nodeslist file
|
||
def get_nodes_list(nodeslist_file):
|
||
global NODES
|
||
if NODES is not None:
|
||
return NODES
|
||
fd = open(nodeslist_file, 'r')
|
||
NODES = json.load(fd)
|
||
fd.close()
|
||
return NODES
|
||
|
rudder-server-relay/SOURCES/relay-api/relay_api/shared_files.py | ||
---|---|---|
from relay_api.common import *
|
||
|
||
import base64
|
||
import re
|
||
import os
|
||
import datetime
|
||
import requests
|
||
|
||
# disable ssl warning on rudder connection in all the possible ways
|
||
try:
|
||
import urllib3
|
||
urllib3.disable_warnings()
|
||
except:
|
||
pass
|
||
|
||
try:
|
||
requests.packages.urllib3.disable_warnings()
|
||
except:
|
||
pass
|
||
|
||
from pprint import pprint
|
||
from Crypto.Hash import *
|
||
from Crypto.Signature import PKCS1_v1_5
|
||
from Crypto.PublicKey import RSA
|
||
|
||
|
||
SIGNATURE_FORMAT="header=rudder-signature-v1"
|
||
METADATA_EXTENSION=".metadata"
|
||
|
||
# convert a byte string to hexadecimal representation
|
||
toHex = lambda x:"".join([hex(ord(c))[2:].zfill(2) for c in x])
|
||
|
||
# convert an hexadecimal string to a byte string
|
||
toBin = lambda x:"".join([chr(int(x[c:c+2],16)) for c in range(0,len(x),2)])
|
||
|
||
# Parse a ttl string of the form "1day 2hours 3minute 4seconds"
|
||
# It can be abreviated to thos form "5h 3s"
|
||
# If it is a pure integer it is considered to be seconds
|
||
# Returns a timedelta object
|
||
def parse_ttl(string):
|
||
m = re.match(r'^\s*(\d+)\s*$', string)
|
||
if m:
|
||
days = 0
|
||
seconds = int(m.group(1))
|
||
else:
|
||
daymatch = r'(\d+)d(?:ays?)?'
|
||
hourmatch = r'(\d+)h(?:ours?)?'
|
||
minmatch = r'(\d+)m(?:inutes?)?'
|
||
secmatch = r'(\d+)s(?:econds?)?'
|
||
match = r'^\s*(?:' + daymatch + r'\s*)?(?:' + hourmatch + r'\s*)?(?:' + minmatch + r'\s*)?(?:' + secmatch + r'\s*)?$'
|
||
m = re.match(match, string)
|
||
if m:
|
||
days = 0
|
||
seconds = 0
|
||
if m.group(1) is not None:
|
||
days = int(m.group(1))
|
||
if m.group(2) is not None:
|
||
seconds += 3600 * int(m.group(2))
|
||
if m.group(3) is not None:
|
||
seconds += 60 * int(m.group(3))
|
||
if m.group(4) is not None:
|
||
seconds += int(m.group(4))
|
||
else:
|
||
raise ValueError("ERROR invalid TTL specification:" + string)
|
||
return datetime.timedelta(days, seconds)
|
||
|
||
|
||
# Extract the signature header from a data stream
|
||
# The header is delimited by an empty line
|
||
def get_header(data_stream):
|
||
# format parsing
|
||
line = data_stream.readline()
|
||
if line.rstrip() != SIGNATURE_FORMAT:
|
||
raise ValueError("ERROR unknown signature format: " + str(line))
|
||
header = line
|
||
while True:
|
||
line = data_stream.readline()
|
||
if line == "\n":
|
||
return header
|
||
header += line
|
||
# the return is just above
|
||
|
||
|
||
# Extract informations from header
|
||
def parse_header(header):
|
||
data = {}
|
||
for line in header.rstrip().split():
|
||
m = re.match(r"(\w+)\s*=\s*(.*)", line)
|
||
if m:
|
||
data[m.group(1)] = m.group(2)
|
||
else:
|
||
raise ValueError("ERROR invalid format: " + line)
|
||
return data
|
||
|
||
|
||
# Extract a public key object from headers
|
||
def get_pubkey(header_info):
|
||
# validate header content first
|
||
if 'digest' not in header_info or 'pubkey' not in header_info:
|
||
raise ValueError("ERROR incomplete header, missing digest or public key")
|
||
pem = "-----BEGIN RSA PRIVATE KEY-----\n" + header_info['pubkey'] + "\n-----END RSA PRIVATE KEY-----\n"
|
||
return RSA.importKey(pem)
|
||
|
||
|
||
# Create expiry header line
|
||
def expiry_line(header_info):
|
||
if 'ttl' not in header_info:
|
||
raise ValueError("ERROR: No TTL provided")
|
||
else:
|
||
ttl = parse_ttl(header_info['ttl'])
|
||
expires = datetime.datetime.utcnow() + ttl # we take utcnow because we write a unix timestamp
|
||
timestamp = int((expires - datetime.datetime(1970, 1, 1)).total_seconds()) # convert to unix timestamp
|
||
return "expires=" + str(timestamp) + "\n"
|
||
|
||
|
||
# Hash a message with a given algorithm
|
||
# Returns a hash object
|
||
def get_hash(algorithm, message):
|
||
if algorithm == "sha1":
|
||
h=SHA.new(message)
|
||
elif algorithm == "sha256":
|
||
h=SHA256.new(message)
|
||
elif algorithm == "sha512":
|
||
h=SHA512.new(message)
|
||
else:
|
||
raise ValueError("ERROR unknown key hash type: " + str(algorithm))
|
||
return h
|
||
|
||
|
||
# Validate that a given public key matches the provided hash
|
||
# The public key is a key object and the hash is of the form 'algorithm:kex_value'
|
||
def validate_key(pubkey, keyhash):
|
||
try:
|
||
(keyhash_type, keyhash_value) = keyhash.split(":",1)
|
||
except:
|
||
raise ValueError("ERROR invalid key hash, it should be 'type:value': " + keyhash)
|
||
pubkey_bin = pubkey.exportKey(format="DER")
|
||
h = get_hash(keyhash_type, pubkey_bin)
|
||
return h.hexdigest() == keyhash_value
|
||
|
||
# Validate that a message has been properly signed by the given key
|
||
# The public key is a key object, algorithm is the hash algorithm and digest the hex signature
|
||
# The key algorithm will always be RSA because is is loaded as such
|
||
# Returns a booleas for the validity and the message hash to avoid computing it twice
|
||
def validate_message(message, pubkey, algorithm, digest):
|
||
h = get_hash(algorithm, message)
|
||
cipher = PKCS1_v1_5.new(pubkey)
|
||
return (cipher.verify(h, toBin(digest)), h.hexdigest())
|
||
|
||
|
||
# Find in which directory a shared file should be stored
|
||
def file_directory(shared_path, nodes, my_uuid, target_uuid, source_uuid, file_id):
|
||
if not re.match(r"^[\w\-]+$", file_id):
|
||
raise ValueError("ERROR file_id must be an identifier [A-z0-9_-]: " + str(file_id))
|
||
route_path = '/shared-files/'.join(node_route(nodes, my_uuid, target_uuid))
|
||
return shared_path + "/" + route_path + "/files/" + source_uuid
|
||
|
||
|
||
# Returns the stored hash from the metadata file
|
||
def get_metadata_hash(metadata_file):
|
||
fd = open(metadata_file, 'r')
|
||
line = fd.readline().rstrip()
|
||
if line != SIGNATURE_FORMAT:
|
||
fd.close()
|
||
raise ValueError("ERROR invalid storage: " + line)
|
||
while True:
|
||
line = fd.readline().rstrip()
|
||
m = re.match(r"(\w+)\s*=\s*(.*)", line)
|
||
if m:
|
||
if m.group(1) == "hash_value":
|
||
fd.close()
|
||
return m.group(2)
|
||
else:
|
||
fd.close()
|
||
raise ValueError("ERROR invalid storage: " + line)
|
||
|
||
|
||
# =====================
|
||
# Manage PUT API call
|
||
# =====================
|
||
# Parameters:
|
||
# - target_uuid where to send the file to
|
||
# - source_uuid who sent the file
|
||
# - file_id under which name to store the file
|
||
# - data the file content
|
||
# - nodes the content of the nodes_list file
|
||
# - my_uuid uuid of the current relay (self)
|
||
# - shared_path the shared-files directory path
|
||
# Returns the full path of the created file
|
||
def shared_files_put(target_uuid, source_uuid, file_id, data_stream, nodes, my_uuid, shared_path):
|
||
header = get_header(data_stream)
|
||
info = parse_header(header)
|
||
|
||
# extract information
|
||
pubkey = get_pubkey(info)
|
||
if source_uuid not in nodes:
|
||
raise ValueError("ERROR unknown source node: " + str(source_uuid))
|
||
if "keyhash" not in nodes[source_uuid]:
|
||
raise ValueError("ERROR invalid nodes file on the server for " + source_uuid)
|
||
keyhash = nodes[source_uuid]["keyhash"]
|
||
|
||
# validate key
|
||
if not validate_key(pubkey, keyhash):
|
||
raise ValueError("ERROR invalid public key or not matching UUID")
|
||
|
||
# validate message
|
||
message = data_stream.read()
|
||
(validated, message_hash) = validate_message(message, pubkey, info['algorithm'], info['digest'])
|
||
if not validated:
|
||
raise ValueError("ERROR invalid signature")
|
||
|
||
# add headers
|
||
header += expiry_line(info)
|
||
header += "hash_value=" + message_hash + "\n"
|
||
|
||
# where to store file
|
||
path = file_directory(shared_path, nodes, my_uuid, target_uuid, source_uuid, file_id)
|
||
filename = path + "/" + file_id
|
||
|
||
# write data & metadata
|
||
try:
|
||
os.makedirs(path, 0750)
|
||
except:
|
||
pass # makedirs fails if the directory exists
|
||
fd = open(filename, 'w')
|
||
fd.write(message)
|
||
fd.close()
|
||
fd = open(filename + METADATA_EXTENSION, 'w')
|
||
fd.write(header)
|
||
fd.close()
|
||
return filename
|
||
|
||
|
||
# ======================
|
||
# Forward PUT API call
|
||
# ======================
|
||
# Parameters:
|
||
# - stream stream of posted data
|
||
# - url where to forward to
|
||
# Returns the full path of the created file
|
||
def shared_files_put_forward(stream, url):
|
||
# This is needed to make requests know the length
|
||
# It will avoid streaming the content which would make the next hop fail
|
||
stream.len = stream.limit
|
||
req = requests.put(url, data=stream, verify=False)
|
||
if req.status_code == 200:
|
||
return req.text
|
||
else:
|
||
raise ValueError("Upstream Error: " + req.text)
|
||
|
||
# ======================
|
||
# Manage HEAD API call
|
||
# ======================
|
||
# Parameters:
|
||
# - target_uuid where to send the file to
|
||
# - source_uuid who sent the file
|
||
# - file_id under which name to store the file
|
||
# - file_hash the hash to compare with
|
||
# - nodes the content of the nodes_list file
|
||
# - my_uuid uuid of the current relay (self)
|
||
# - shared_path the shared-files directory path
|
||
# Returns true of false
|
||
def shared_files_head(target_uuid, source_uuid, file_id, file_hash, nodes, my_uuid, shared_path):
|
||
# where to find file
|
||
path = file_directory(shared_path, nodes, my_uuid, target_uuid, source_uuid, file_id)
|
||
filename = path + "/" + file_id
|
||
metadata = filename + METADATA_EXTENSION
|
||
|
||
# check if the file and signature exist
|
||
if not os.path.isfile(filename) or not os.path.isfile(metadata):
|
||
return False
|
||
|
||
# check hash from metadata
|
||
hash_value = get_metadata_hash(metadata)
|
||
return hash_value == file_hash
|
||
|
||
# =======================
|
||
# Forward HEAD API call
|
||
# =======================
|
||
# Parameters:
|
||
# - url where to forward to
|
||
# Returns true of false
|
||
def shared_files_head_forward(url):
|
||
req = requests.head(url, verify=False)
|
||
if req.status_code == 200:
|
||
return True
|
||
if req.status_code == 404:
|
||
return False
|
||
raise ValueError("ERROR from server:" + str(req.status_code))
|
rudder-server-relay/SOURCES/relay-api/relay_api/views.py | ||
---|---|---|
from relay_api import app
|
||
from relay_api.shared_files import shared_files_put, shared_files_head, shared_files_put_forward, shared_files_head_forward
|
||
from relay_api.common import *
|
||
|
||
from flask import Flask, jsonify, request, abort, make_response
|
||
from StringIO import StringIO
|
||
import traceback
|
||
|
||
from pprint import pprint
|
||
|
||
NODESLIST_FILE = "/var/rudder/cfengine-community/inputs/distributePolicy/1.0/nodeslist.json"
|
||
UUID_FILE = '/opt/rudder/etc/uuid.hive'
|
||
API_DEBUGINFO = True
|
||
POLICY_SERVER_FILE = "/var/rudder/cfengine-community/policy_server.dat"
|
||
SHARED_FILES_PATH = "/var/rudder/shared-files"
|
||
|
||
#################
|
||
# API functions #
|
||
#################
|
||
|
||
@app.route('/shared-files/<string:target_uuid>/<string:source_uuid>/<string:file_id>', methods=['PUT'])
|
||
def put_file(target_uuid, source_uuid, file_id):
|
||
try:
|
||
nodes = get_nodes_list(NODESLIST_FILE)
|
||
if target_uuid not in nodes:
|
||
# forward the file if the node is unknown
|
||
policy_server = get_file_content(POLICY_SERVER_FILE)
|
||
if policy_server == "root":
|
||
return format_response("Unknown UUID: "+target_uuid, 404)
|
||
else:
|
||
url = "https://"+policy_server+"/rudder/relay-api/shared-files/" + target_uuid + "/" + source_uuid + "/" + file_id + "?hash=" + file_hash
|
||
res = shared_files_put_forward(request.stream, url)
|
||
else:
|
||
# process the file if it is known
|
||
my_uuid = get_file_content(UUID_FILE)
|
||
filename = shared_files_put(target_uuid, source_uuid, file_id, request.stream, nodes, my_uuid, SHARED_FILES_PATH)
|
||
if API_DEBUGINFO:
|
||
res = "OK\nWritten to: " + filename + "\n"
|
||
else:
|
||
res = "OK\n"
|
||
return format_response(res)
|
||
except Exception as e:
|
||
return format_error(e, API_DEBUGINFO)
|
||
|
||
|
||
@app.route('/shared-files/<string:target_uuid>/<string:source_uuid>/<string:file_id>', methods=['HEAD'])
|
||
def head_file(target_uuid, source_uuid, file_id):
|
||
try:
|
||
nodes = get_nodes_list(NODESLIST_FILE)
|
||
file_hash = request.args.get('hash', '')
|
||
if target_uuid not in nodes:
|
||
# forward the request if the node is unknown
|
||
policy_server = get_file_content(POLICY_SERVER_FILE)
|
||
if policy_server == "root":
|
||
return format_response("Unknown UUID: "+target_uuid, 404)
|
||
else:
|
||
url = "https://"+policy_server+"/rudder/relay-api/shared-files/" + target_uuid + "/" + source_uuid + "/" + file_id + "?hash=" + file_hash
|
||
res = shared_files_head_forward(url)
|
||
else:
|
||
# process the request if it is known
|
||
my_uuid = get_file_content(UUID_FILE)
|
||
res = shared_files_head(target_uuid, source_uuid, file_id, file_hash, nodes, my_uuid, SHARED_FILES_PATH)
|
||
if res:
|
||
return format_response("", 200)
|
||
else:
|
||
return format_response("", 404)
|
||
except Exception as e:
|
||
print(traceback.format_exc())
|
||
return format_error(e, API_DEBUGINFO)
|
||
|
||
|
||
# main
|
||
if __name__ == '__main__':
|
||
app.run(debug = True)
|
||
|
rudder-server-relay/SOURCES/relay-api/requirements.txt | ||
---|---|---|
flask
|
||
requests
|
||
pycrypto
|
rudder-server-relay/SOURCES/relay-api/run.py | ||
---|---|---|
#!flask/bin/python
|
||
|
||
# This file is only present for development/ local test and should not be used in production
|
||
# To deploy ncf api you should use it with a virtual environment and a wsgi file
|
||
# an example of this is available in ncf-api-virualenv package
|
||
# Virtualenv defined should be in local directory flask
|
||
|
||
import requests
|
||
from relay_api import app
|
||
|
||
app.run(debug = True)
|
rudder-server-relay/SOURCES/relay-api/virtualenv.py | ||
---|---|---|
#!/usr/bin/env python
|
||
"""Create a "virtual" Python installation
|
||
"""
|
||
|
||
# If you change the version here, change it in setup.py
|
||
# and docs/conf.py as well.
|
||
__version__ = "1.9.1" # following best practices
|
||
virtualenv_version = __version__ # legacy, again
|
||
|
||
import base64
|
||
import sys
|
||
import os
|
||
import codecs
|
||
import optparse
|
||
import re
|
||
import shutil
|
||
import logging
|
||
import tempfile
|
||
import zlib
|
||
import errno
|
||
import glob
|
||
import distutils.sysconfig
|
||
from distutils.util import strtobool
|
||
import struct
|
||
import subprocess
|
||
|
||
if sys.version_info < (2, 5):
|
||
print('ERROR: %s' % sys.exc_info()[1])
|
||
print('ERROR: this script requires Python 2.5 or greater.')
|
||
sys.exit(101)
|
||
|
||
try:
|
||
set
|
||
except NameError:
|
||
from sets import Set as set
|
||
try:
|
||
basestring
|
||
except NameError:
|
||
basestring = str
|
||
|
||
try:
|
||
import ConfigParser
|
||
except ImportError:
|
||
import configparser as ConfigParser
|
||
|
||
join = os.path.join
|
||
py_version = 'python%s.%s' % (sys.version_info[0], sys.version_info[1])
|
||
|
||
is_jython = sys.platform.startswith('java')
|
||
is_pypy = hasattr(sys, 'pypy_version_info')
|
||
is_win = (sys.platform == 'win32')
|
||
is_cygwin = (sys.platform == 'cygwin')
|
||
is_darwin = (sys.platform == 'darwin')
|
||
abiflags = getattr(sys, 'abiflags', '')
|
||
|
||
user_dir = os.path.expanduser('~')
|
||
if is_win:
|
||
default_storage_dir = os.path.join(user_dir, 'virtualenv')
|
||
else:
|
||
default_storage_dir = os.path.join(user_dir, '.virtualenv')
|
||
default_config_file = os.path.join(default_storage_dir, 'virtualenv.ini')
|
||
|
||
if is_pypy:
|
||
expected_exe = 'pypy'
|
||
elif is_jython:
|
||
expected_exe = 'jython'
|
||
else:
|
||
expected_exe = 'python'
|
||
|
||
|
||
REQUIRED_MODULES = ['os', 'posix', 'posixpath', 'nt', 'ntpath', 'genericpath',
|
||
'fnmatch', 'locale', 'encodings', 'codecs',
|
||
'stat', 'UserDict', 'readline', 'copy_reg', 'types',
|
||
're', 'sre', 'sre_parse', 'sre_constants', 'sre_compile',
|
||
'zlib']
|
||
|
||
REQUIRED_FILES = ['lib-dynload', 'config']
|
||
|
||
majver, minver = sys.version_info[:2]
|
||
if majver == 2:
|
||
if minver >= 6:
|
||
REQUIRED_MODULES.extend(['warnings', 'linecache', '_abcoll', 'abc'])
|
||
if minver >= 7:
|
||
REQUIRED_MODULES.extend(['_weakrefset'])
|
||
if minver <= 3:
|
||
REQUIRED_MODULES.extend(['sets', '__future__'])
|
||
elif majver == 3:
|
||
# Some extra modules are needed for Python 3, but different ones
|
||
# for different versions.
|
||
REQUIRED_MODULES.extend(['_abcoll', 'warnings', 'linecache', 'abc', 'io',
|
||
'_weakrefset', 'copyreg', 'tempfile', 'random',
|
||
'__future__', 'collections', 'keyword', 'tarfile',
|
||
'shutil', 'struct', 'copy', 'tokenize', 'token',
|
||
'functools', 'heapq', 'bisect', 'weakref',
|
||
'reprlib'])
|
||
if minver >= 2:
|
||
REQUIRED_FILES[-1] = 'config-%s' % majver
|
||
if minver == 3:
|
||
import sysconfig
|
||
platdir = sysconfig.get_config_var('PLATDIR')
|
||
REQUIRED_FILES.append(platdir)
|
||
# The whole list of 3.3 modules is reproduced below - the current
|
||
# uncommented ones are required for 3.3 as of now, but more may be
|
||
# added as 3.3 development continues.
|
||
REQUIRED_MODULES.extend([
|
||
#"aifc",
|
||
#"antigravity",
|
||
#"argparse",
|
||
#"ast",
|
||
#"asynchat",
|
||
#"asyncore",
|
||
"base64",
|
||
#"bdb",
|
||
#"binhex",
|
||
#"bisect",
|
||
#"calendar",
|
||
#"cgi",
|
||
#"cgitb",
|
||
#"chunk",
|
||
#"cmd",
|
||
#"codeop",
|
||
#"code",
|
||
#"colorsys",
|
||
#"_compat_pickle",
|
||
#"compileall",
|
||
#"concurrent",
|
||
#"configparser",
|
||
#"contextlib",
|
||
#"cProfile",
|
||
#"crypt",
|
||
#"csv",
|
||
#"ctypes",
|
||
#"curses",
|
||
#"datetime",
|
||
#"dbm",
|
||
#"decimal",
|
||
#"difflib",
|
||
#"dis",
|
||
#"doctest",
|
||
#"dummy_threading",
|
||
"_dummy_thread",
|
||
#"email",
|
||
#"filecmp",
|
||
#"fileinput",
|
||
#"formatter",
|
||
#"fractions",
|
||
#"ftplib",
|
||
#"functools",
|
||
#"getopt",
|
||
#"getpass",
|
||
#"gettext",
|
||
#"glob",
|
||
#"gzip",
|
||
"hashlib",
|
||
#"heapq",
|
||
"hmac",
|
||
#"html",
|
||
#"http",
|
||
#"idlelib",
|
||
#"imaplib",
|
||
#"imghdr",
|
||
"imp",
|
||
"importlib",
|
||
#"inspect",
|
||
#"json",
|
||
#"lib2to3",
|
||
#"logging",
|
||
#"macpath",
|
||
#"macurl2path",
|
||
#"mailbox",
|
||
#"mailcap",
|
||
#"_markupbase",
|
||
#"mimetypes",
|
||
#"modulefinder",
|
||
#"multiprocessing",
|
||
#"netrc",
|
||
#"nntplib",
|
||
#"nturl2path",
|
||
#"numbers",
|
||
#"opcode",
|
||
#"optparse",
|
||
#"os2emxpath",
|
||
#"pdb",
|
||
#"pickle",
|
||
#"pickletools",
|
||
#"pipes",
|
||
#"pkgutil",
|
||
#"platform",
|
||
#"plat-linux2",
|
||
#"plistlib",
|
||
#"poplib",
|
||
#"pprint",
|
||
#"profile",
|
||
#"pstats",
|
||
#"pty",
|
||
#"pyclbr",
|
||
#"py_compile",
|
||
#"pydoc_data",
|
||
#"pydoc",
|
||
#"_pyio",
|
||
#"queue",
|
||
#"quopri",
|
||
#"reprlib",
|
||
"rlcompleter",
|
||
#"runpy",
|
||
#"sched",
|
||
#"shelve",
|
||
#"shlex",
|
||
#"smtpd",
|
||
#"smtplib",
|
||
#"sndhdr",
|
||
#"socket",
|
||
#"socketserver",
|
||
#"sqlite3",
|
||
#"ssl",
|
||
#"stringprep",
|
||
#"string",
|
||
#"_strptime",
|
||
#"subprocess",
|
||
#"sunau",
|
||
#"symbol",
|
||
#"symtable",
|
||
#"sysconfig",
|
||
#"tabnanny",
|
||
#"telnetlib",
|
||
#"test",
|
||
#"textwrap",
|
||
#"this",
|
||
#"_threading_local",
|
||
#"threading",
|
||
#"timeit",
|
||
#"tkinter",
|
||
#"tokenize",
|
||
#"token",
|
||
#"traceback",
|
||
#"trace",
|
||
#"tty",
|
||
#"turtledemo",
|
||
#"turtle",
|
||
#"unittest",
|
||
#"urllib",
|
||
#"uuid",
|
||
#"uu",
|
||
#"wave",
|
||
#"weakref",
|
||
#"webbrowser",
|
||
#"wsgiref",
|
||
#"xdrlib",
|
||
#"xml",
|
||
#"xmlrpc",
|
||
#"zipfile",
|
||
])
|
||
|
||
if is_pypy:
|
||
# these are needed to correctly display the exceptions that may happen
|
||
# during the bootstrap
|
||
REQUIRED_MODULES.extend(['traceback', 'linecache'])
|
||
|
||
class Logger(object):
|
||
|
||
"""
|
||
Logging object for use in command-line script. Allows ranges of
|
||
levels, to avoid some redundancy of displayed information.
|
||
"""
|
||
|
||
DEBUG = logging.DEBUG
|
||
INFO = logging.INFO
|
||
NOTIFY = (logging.INFO+logging.WARN)/2
|
||
WARN = WARNING = logging.WARN
|
||
ERROR = logging.ERROR
|
||
FATAL = logging.FATAL
|
||
|
||
LEVELS = [DEBUG, INFO, NOTIFY, WARN, ERROR, FATAL]
|
||
|
||
def __init__(self, consumers):
|
||
self.consumers = consumers
|
||
self.indent = 0
|
||
self.in_progress = None
|
||
self.in_progress_hanging = False
|
||
|
||
def debug(self, msg, *args, **kw):
|
||
self.log(self.DEBUG, msg, *args, **kw)
|
||
def info(self, msg, *args, **kw):
|
||
self.log(self.INFO, msg, *args, **kw)
|
||
def notify(self, msg, *args, **kw):
|
||
self.log(self.NOTIFY, msg, *args, **kw)
|
||
def warn(self, msg, *args, **kw):
|
||
self.log(self.WARN, msg, *args, **kw)
|
||
def error(self, msg, *args, **kw):
|
||
self.log(self.ERROR, msg, *args, **kw)
|
||
def fatal(self, msg, *args, **kw):
|
||
self.log(self.FATAL, msg, *args, **kw)
|
||
def log(self, level, msg, *args, **kw):
|
||
if args:
|
||
if kw:
|
||
raise TypeError(
|
||
"You may give positional or keyword arguments, not both")
|
||
args = args or kw
|
||
rendered = None
|
||
for consumer_level, consumer in self.consumers:
|
||
if self.level_matches(level, consumer_level):
|
||
if (self.in_progress_hanging
|
||
and consumer in (sys.stdout, sys.stderr)):
|
||
self.in_progress_hanging = False
|
||
sys.stdout.write('\n')
|
||
sys.stdout.flush()
|
||
if rendered is None:
|
||
if args:
|
||
rendered = msg % args
|
||
else:
|
||
rendered = msg
|
||
rendered = ' '*self.indent + rendered
|
||
if hasattr(consumer, 'write'):
|
||
consumer.write(rendered+'\n')
|
||
else:
|
||
consumer(rendered)
|
||
|
||
def start_progress(self, msg):
|
||
assert not self.in_progress, (
|
||
"Tried to start_progress(%r) while in_progress %r"
|
||
% (msg, self.in_progress))
|
||
if self.level_matches(self.NOTIFY, self._stdout_level()):
|
||
sys.stdout.write(msg)
|
||
sys.stdout.flush()
|
||
self.in_progress_hanging = True
|
||
else:
|
||
self.in_progress_hanging = False
|
||
self.in_progress = msg
|
||
|
||
def end_progress(self, msg='done.'):
|
||
assert self.in_progress, (
|
||
"Tried to end_progress without start_progress")
|
||
if self.stdout_level_matches(self.NOTIFY):
|
||
if not self.in_progress_hanging:
|
||
# Some message has been printed out since start_progress
|
||
sys.stdout.write('...' + self.in_progress + msg + '\n')
|
||
sys.stdout.flush()
|
||
else:
|
||
sys.stdout.write(msg + '\n')
|
||
sys.stdout.flush()
|
||
self.in_progress = None
|
||
self.in_progress_hanging = False
|
||
|
||
def show_progress(self):
|
||
"""If we are in a progress scope, and no log messages have been
|
||
shown, write out another '.'"""
|
||
if self.in_progress_hanging:
|
||
sys.stdout.write('.')
|
||
sys.stdout.flush()
|
||
|
||
def stdout_level_matches(self, level):
|
||
"""Returns true if a message at this level will go to stdout"""
|
||
return self.level_matches(level, self._stdout_level())
|
||
|
||
def _stdout_level(self):
|
||
"""Returns the level that stdout runs at"""
|
||
for level, consumer in self.consumers:
|
||
if consumer is sys.stdout:
|
||
return level
|
||
return self.FATAL
|
||
|
||
def level_matches(self, level, consumer_level):
|
||
"""
|
||
>>> l = Logger([])
|
||
>>> l.level_matches(3, 4)
|
||
False
|
||
>>> l.level_matches(3, 2)
|
||
True
|
||
>>> l.level_matches(slice(None, 3), 3)
|
||
False
|
||
>>> l.level_matches(slice(None, 3), 2)
|
||
True
|
||
>>> l.level_matches(slice(1, 3), 1)
|
||
True
|
||
>>> l.level_matches(slice(2, 3), 1)
|
||
False
|
||
"""
|
||
if isinstance(level, slice):
|
||
start, stop = level.start, level.stop
|
||
if start is not None and start > consumer_level:
|
||
return False
|
||
if stop is not None and stop <= consumer_level:
|
||
return False
|
||
return True
|
||
else:
|
||
return level >= consumer_level
|
||
|
||
#@classmethod
|
||
def level_for_integer(cls, level):
|
||
levels = cls.LEVELS
|
||
if level < 0:
|
||
return levels[0]
|
||
if level >= len(levels):
|
||
return levels[-1]
|
||
return levels[level]
|
||
|
||
level_for_integer = classmethod(level_for_integer)
|
||
|
||
# create a silent logger just to prevent this from being undefined
|
||
# will be overridden with requested verbosity main() is called.
|
||
logger = Logger([(Logger.LEVELS[-1], sys.stdout)])
|
||
|
||
def mkdir(path):
|
||
if not os.path.exists(path):
|
||
logger.info('Creating %s', path)
|
||
os.makedirs(path)
|
||
else:
|
||
logger.info('Directory %s already exists', path)
|
||
|
||
def copyfileordir(src, dest):
|
||
if os.path.isdir(src):
|
||
shutil.copytree(src, dest, True)
|
||
else:
|
||
shutil.copy2(src, dest)
|
||
|
||
def copyfile(src, dest, symlink=True):
|
||
if not os.path.exists(src):
|
||
# Some bad symlink in the src
|
||
logger.warn('Cannot find file %s (bad symlink)', src)
|
||
return
|
||
if os.path.exists(dest):
|
||
logger.debug('File %s already exists', dest)
|
||
return
|
||
if not os.path.exists(os.path.dirname(dest)):
|
||
logger.info('Creating parent directories for %s' % os.path.dirname(dest))
|
||
os.makedirs(os.path.dirname(dest))
|
||
if not os.path.islink(src):
|
||
srcpath = os.path.abspath(src)
|
||
else:
|
||
srcpath = os.readlink(src)
|
||
if symlink and hasattr(os, 'symlink') and not is_win:
|
||
logger.info('Symlinking %s', dest)
|
||
try:
|
||
os.symlink(srcpath, dest)
|
||
except (OSError, NotImplementedError):
|
||
logger.info('Symlinking failed, copying to %s', dest)
|
||
copyfileordir(src, dest)
|
||
else:
|
||
logger.info('Copying to %s', dest)
|
||
copyfileordir(src, dest)
|
||
|
||
def writefile(dest, content, overwrite=True):
|
||
if not os.path.exists(dest):
|
||
logger.info('Writing %s', dest)
|
||
f = open(dest, 'wb')
|
||
f.write(content.encode('utf-8'))
|
||
f.close()
|
||
return
|
||
else:
|
||
f = open(dest, 'rb')
|
||
c = f.read()
|
||
f.close()
|
||
if c != content.encode("utf-8"):
|
||
if not overwrite:
|
||
logger.notify('File %s exists with different content; not overwriting', dest)
|
||
return
|
||
logger.notify('Overwriting %s with new content', dest)
|
||
f = open(dest, 'wb')
|
||
f.write(content.encode('utf-8'))
|
||
f.close()
|
||
else:
|
||
logger.info('Content %s already in place', dest)
|
||
|
||
def rmtree(dir):
|
||
if os.path.exists(dir):
|
||
logger.notify('Deleting tree %s', dir)
|
||
shutil.rmtree(dir)
|
||
else:
|
||
logger.info('Do not need to delete %s; already gone', dir)
|
||
|
||
def make_exe(fn):
|
||
if hasattr(os, 'chmod'):
|
||
oldmode = os.stat(fn).st_mode & 0xFFF # 0o7777
|
||
newmode = (oldmode | 0x16D) & 0xFFF # 0o555, 0o7777
|
||
os.chmod(fn, newmode)
|
||
logger.info('Changed mode of %s to %s', fn, oct(newmode))
|
||
|
||
def _find_file(filename, dirs):
|
||
for dir in reversed(dirs):
|
||
files = glob.glob(os.path.join(dir, filename))
|
||
if files and os.path.isfile(files[0]):
|
||
return True, files[0]
|
||
return False, filename
|
||
|
||
def _install_req(py_executable, unzip=False, distribute=False,
|
||
search_dirs=None, never_download=False):
|
||
|
||
if search_dirs is None:
|
||
search_dirs = file_search_dirs()
|
||
|
||
if not distribute:
|
||
egg_path = 'setuptools-*-py%s.egg' % sys.version[:3]
|
||
found, egg_path = _find_file(egg_path, search_dirs)
|
||
project_name = 'setuptools'
|
||
bootstrap_script = EZ_SETUP_PY
|
||
tgz_path = None
|
||
else:
|
||
# Look for a distribute egg (these are not distributed by default,
|
||
# but can be made available by the user)
|
||
egg_path = 'distribute-*-py%s.egg' % sys.version[:3]
|
||
found, egg_path = _find_file(egg_path, search_dirs)
|
||
project_name = 'distribute'
|
||
if found:
|
||
tgz_path = None
|
||
bootstrap_script = DISTRIBUTE_FROM_EGG_PY
|
||
else:
|
||
# Fall back to sdist
|
||
# NB: egg_path is not None iff tgz_path is None
|
||
# iff bootstrap_script is a generic setup script accepting
|
||
# the standard arguments.
|
||
egg_path = None
|
||
tgz_path = 'distribute-*.tar.gz'
|
||
found, tgz_path = _find_file(tgz_path, search_dirs)
|
||
bootstrap_script = DISTRIBUTE_SETUP_PY
|
||
|
||
if is_jython and os._name == 'nt':
|
||
# Jython's .bat sys.executable can't handle a command line
|
||
# argument with newlines
|
||
fd, ez_setup = tempfile.mkstemp('.py')
|
||
os.write(fd, bootstrap_script)
|
||
os.close(fd)
|
||
cmd = [py_executable, ez_setup]
|
||
else:
|
||
cmd = [py_executable, '-c', bootstrap_script]
|
||
if unzip and egg_path:
|
||
cmd.append('--always-unzip')
|
||
env = {}
|
||
remove_from_env = ['__PYVENV_LAUNCHER__']
|
||
if logger.stdout_level_matches(logger.DEBUG) and egg_path:
|
||
cmd.append('-v')
|
||
|
||
old_chdir = os.getcwd()
|
||
if egg_path is not None and os.path.exists(egg_path):
|
||
logger.info('Using existing %s egg: %s' % (project_name, egg_path))
|
||
cmd.append(egg_path)
|
||
if os.environ.get('PYTHONPATH'):
|
||
env['PYTHONPATH'] = egg_path + os.path.pathsep + os.environ['PYTHONPATH']
|
||
else:
|
||
env['PYTHONPATH'] = egg_path
|
||
elif tgz_path is not None and os.path.exists(tgz_path):
|
||
# Found a tgz source dist, let's chdir
|
||
logger.info('Using existing %s egg: %s' % (project_name, tgz_path))
|
||
os.chdir(os.path.dirname(tgz_path))
|
||
# in this case, we want to be sure that PYTHONPATH is unset (not
|
||
# just empty, really unset), else CPython tries to import the
|
||
# site.py that it's in virtualenv_support
|
||
remove_from_env.append('PYTHONPATH')
|
||
elif never_download:
|
||
logger.fatal("Can't find any local distributions of %s to install "
|
||
"and --never-download is set. Either re-run virtualenv "
|
||
"without the --never-download option, or place a %s "
|
||
"distribution (%s) in one of these "
|
||
"locations: %r" % (project_name, project_name,
|
||
egg_path or tgz_path,
|
||
search_dirs))
|
||
sys.exit(1)
|
||
elif egg_path:
|
||
logger.info('No %s egg found; downloading' % project_name)
|
||
cmd.extend(['--always-copy', '-U', project_name])
|
||
else:
|
||
logger.info('No %s tgz found; downloading' % project_name)
|
||
logger.start_progress('Installing %s...' % project_name)
|
||
logger.indent += 2
|
||
cwd = None
|
||
if project_name == 'distribute':
|
||
env['DONT_PATCH_SETUPTOOLS'] = 'true'
|
||
|
||
def _filter_ez_setup(line):
|
||
return filter_ez_setup(line, project_name)
|
||
|
||
if not os.access(os.getcwd(), os.W_OK):
|
||
cwd = tempfile.mkdtemp()
|
||
if tgz_path is not None and os.path.exists(tgz_path):
|
||
# the current working dir is hostile, let's copy the
|
||
# tarball to a temp dir
|
||
target = os.path.join(cwd, os.path.split(tgz_path)[-1])
|
||
shutil.copy(tgz_path, target)
|
||
try:
|
||
call_subprocess(cmd, show_stdout=False,
|
||
filter_stdout=_filter_ez_setup,
|
||
extra_env=env,
|
||
remove_from_env=remove_from_env,
|
||
cwd=cwd)
|
||
finally:
|
||
logger.indent -= 2
|
||
logger.end_progress()
|
||
if cwd is not None:
|
||
shutil.rmtree(cwd)
|
||
if os.getcwd() != old_chdir:
|
||
os.chdir(old_chdir)
|
||
if is_jython and os._name == 'nt':
|
||
os.remove(ez_setup)
|
||
|
||
def file_search_dirs():
|
||
here = os.path.dirname(os.path.abspath(__file__))
|
||
dirs = ['.', here,
|
||
join(here, 'virtualenv_support')]
|
||
if os.path.splitext(os.path.dirname(__file__))[0] != 'virtualenv':
|
||
# Probably some boot script; just in case virtualenv is installed...
|
||
try:
|
||
import virtualenv
|
||
except ImportError:
|
||
pass
|
||
else:
|
||
dirs.append(os.path.join(os.path.dirname(virtualenv.__file__), 'virtualenv_support'))
|
||
return [d for d in dirs if os.path.isdir(d)]
|
||
|
||
def install_setuptools(py_executable, unzip=False,
|
||
search_dirs=None, never_download=False):
|
||
_install_req(py_executable, unzip,
|
||
search_dirs=search_dirs, never_download=never_download)
|
||
|
||
def install_distribute(py_executable, unzip=False,
|
||
search_dirs=None, never_download=False):
|
||
_install_req(py_executable, unzip, distribute=True,
|
||
search_dirs=search_dirs, never_download=never_download)
|
||
|
||
_pip_re = re.compile(r'^pip-.*(zip|tar.gz|tar.bz2|tgz|tbz)$', re.I)
|
||
def install_pip(py_executable, search_dirs=None, never_download=False):
|
||
if search_dirs is None:
|
||
search_dirs = file_search_dirs()
|
||
|
||
filenames = []
|
||
for dir in search_dirs:
|
||
filenames.extend([join(dir, fn) for fn in os.listdir(dir)
|
||
if _pip_re.search(fn)])
|
||
filenames = [(os.path.basename(filename).lower(), i, filename) for i, filename in enumerate(filenames)]
|
||
filenames.sort()
|
||
filenames = [filename for basename, i, filename in filenames]
|
||
if not filenames:
|
||
filename = 'pip'
|
||
else:
|
||
filename = filenames[-1]
|
||
easy_install_script = 'easy_install'
|
||
if is_win:
|
||
easy_install_script = 'easy_install-script.py'
|
||
# There's two subtle issues here when invoking easy_install.
|
||
# 1. On unix-like systems the easy_install script can *only* be executed
|
||
# directly if its full filesystem path is no longer than 78 characters.
|
||
# 2. A work around to [1] is to use the `python path/to/easy_install foo`
|
||
# pattern, but that breaks if the path contains non-ASCII characters, as
|
||
# you can't put the file encoding declaration before the shebang line.
|
||
# The solution is to use Python's -x flag to skip the first line of the
|
||
# script (and any ASCII decoding errors that may have occurred in that line)
|
||
cmd = [py_executable, '-x', join(os.path.dirname(py_executable), easy_install_script), filename]
|
||
# jython and pypy don't yet support -x
|
||
if is_jython or is_pypy:
|
||
cmd.remove('-x')
|
||
if filename == 'pip':
|
||
if never_download:
|
||
logger.fatal("Can't find any local distributions of pip to install "
|
||
"and --never-download is set. Either re-run virtualenv "
|
||
"without the --never-download option, or place a pip "
|
||
"source distribution (zip/tar.gz/tar.bz2) in one of these "
|
||
"locations: %r" % search_dirs)
|
||
sys.exit(1)
|
||
logger.info('Installing pip from network...')
|
||
else:
|
||
logger.info('Installing existing %s distribution: %s' % (
|
||
os.path.basename(filename), filename))
|
||
logger.start_progress('Installing pip...')
|
||
logger.indent += 2
|
||
def _filter_setup(line):
|
||
return filter_ez_setup(line, 'pip')
|
||
try:
|
||
call_subprocess(cmd, show_stdout=False,
|
||
filter_stdout=_filter_setup)
|
||
finally:
|
||
logger.indent -= 2
|
||
logger.end_progress()
|
||
|
||
def filter_ez_setup(line, project_name='setuptools'):
|
||
if not line.strip():
|
||
return Logger.DEBUG
|
||
if project_name == 'distribute':
|
||
for prefix in ('Extracting', 'Now working', 'Installing', 'Before',
|
||
'Scanning', 'Setuptools', 'Egg', 'Already',
|
||
'running', 'writing', 'reading', 'installing',
|
||
'creating', 'copying', 'byte-compiling', 'removing',
|
||
'Processing'):
|
||
if line.startswith(prefix):
|
||
return Logger.DEBUG
|
||
return Logger.DEBUG
|
||
for prefix in ['Reading ', 'Best match', 'Processing setuptools',
|
||
'Copying setuptools', 'Adding setuptools',
|
||
'Installing ', 'Installed ']:
|
||
if line.startswith(prefix):
|
||
return Logger.DEBUG
|
||
return Logger.INFO
|
||
|
||
|
||
class UpdatingDefaultsHelpFormatter(optparse.IndentedHelpFormatter):
|
||
"""
|
||
Custom help formatter for use in ConfigOptionParser that updates
|
||
the defaults before expanding them, allowing them to show up correctly
|
||
in the help listing
|
||
"""
|
||
def expand_default(self, option):
|
||
if self.parser is not None:
|
||
self.parser.update_defaults(self.parser.defaults)
|
||
return optparse.IndentedHelpFormatter.expand_default(self, option)
|
||
|
||
|
||
class ConfigOptionParser(optparse.OptionParser):
|
||
"""
|
||
Custom option parser which updates its defaults by by checking the
|
||
configuration files and environmental variables
|
||
"""
|
||
def __init__(self, *args, **kwargs):
|
||
self.config = ConfigParser.RawConfigParser()
|
||
self.files = self.get_config_files()
|
||
self.config.read(self.files)
|
||
optparse.OptionParser.__init__(self, *args, **kwargs)
|
||
|
||
def get_config_files(self):
|
||
config_file = os.environ.get('VIRTUALENV_CONFIG_FILE', False)
|
||
if config_file and os.path.exists(config_file):
|
||
return [config_file]
|
||
return [default_config_file]
|
||
|
||
def update_defaults(self, defaults):
|
||
"""
|
||
Updates the given defaults with values from the config files and
|
||
the environ. Does a little special handling for certain types of
|
||
options (lists).
|
||
"""
|
||
# Then go and look for the other sources of configuration:
|
||
config = {}
|
||
# 1. config files
|
||
config.update(dict(self.get_config_section('virtualenv')))
|
||
# 2. environmental variables
|
||
config.update(dict(self.get_environ_vars()))
|
||
# Then set the options with those values
|
||
for key, val in config.items():
|
||
key = key.replace('_', '-')
|
||
if not key.startswith('--'):
|
||
key = '--%s' % key # only prefer long opts
|
||
option = self.get_option(key)
|
||
if option is not None:
|
||
# ignore empty values
|
||
if not val:
|
||
continue
|
||
# handle multiline configs
|
||
if option.action == 'append':
|
||
val = val.split()
|
||
else:
|
||
option.nargs = 1
|
||
if option.action == 'store_false':
|
||
val = not strtobool(val)
|
||
elif option.action in ('store_true', 'count'):
|
||
val = strtobool(val)
|
||
try:
|
||
val = option.convert_value(key, val)
|
||
except optparse.OptionValueError:
|
||
e = sys.exc_info()[1]
|
||
print("An error occured during configuration: %s" % e)
|
||
sys.exit(3)
|
||
defaults[option.dest] = val
|
||
return defaults
|
||
|
||
def get_config_section(self, name):
|
||
"""
|
||
Get a section of a configuration
|
||
"""
|
||
if self.config.has_section(name):
|
||
return self.config.items(name)
|
||
return []
|
||
|
||
def get_environ_vars(self, prefix='VIRTUALENV_'):
|
||
"""
|
||
Returns a generator with all environmental vars with prefix VIRTUALENV
|
||
"""
|
||
for key, val in os.environ.items():
|
||
if key.startswith(prefix):
|
||
yield (key.replace(prefix, '').lower(), val)
|
||
|
||
def get_default_values(self):
|
||
"""
|
||
Overridding to make updating the defaults after instantiation of
|
||
the option parser possible, update_defaults() does the dirty work.
|
||
"""
|
||
if not self.process_default_values:
|
||
# Old, pre-Optik 1.5 behaviour.
|
||
return optparse.Values(self.defaults)
|
||
|
||
defaults = self.update_defaults(self.defaults.copy()) # ours
|
||
for option in self._get_all_options():
|
||
default = defaults.get(option.dest)
|
||
if isinstance(default, basestring):
|
||
opt_str = option.get_opt_string()
|
||
defaults[option.dest] = option.check_value(opt_str, default)
|
||
return optparse.Values(defaults)
|
||
|
||
|
||
def main():
|
||
parser = ConfigOptionParser(
|
||
version=virtualenv_version,
|
||
usage="%prog [OPTIONS] DEST_DIR",
|
||
formatter=UpdatingDefaultsHelpFormatter())
|
||
|
||
parser.add_option(
|
||
'-v', '--verbose',
|
||
action='count',
|
||
dest='verbose',
|
||
default=0,
|
||
help="Increase verbosity")
|
||
|
||
parser.add_option(
|
||
'-q', '--quiet',
|
||
action='count',
|
||
dest='quiet',
|
||
default=0,
|
||
help='Decrease verbosity')
|
||
|
||
parser.add_option(
|
||
'-p', '--python',
|
||
dest='python',
|
||
metavar='PYTHON_EXE',
|
||
help='The Python interpreter to use, e.g., --python=python2.5 will use the python2.5 '
|
||
'interpreter to create the new environment. The default is the interpreter that '
|
||
'virtualenv was installed with (%s)' % sys.executable)
|
||
|
||
parser.add_option(
|
||
'--clear',
|
||
dest='clear',
|
||
action='store_true',
|
||
help="Clear out the non-root install and start from scratch")
|
||
|
||
parser.set_defaults(system_site_packages=False)
|
||
parser.add_option(
|
||
'--no-site-packages',
|
||
dest='system_site_packages',
|
||
action='store_false',
|
||
help="Don't give access to the global site-packages dir to the "
|
||
"virtual environment (default)")
|
||
|
||
parser.add_option(
|
||
'--system-site-packages',
|
||
dest='system_site_packages',
|
||
action='store_true',
|
||
help="Give access to the global site-packages dir to the "
|
||
"virtual environment")
|
||
|
||
parser.add_option(
|
||
'--unzip-setuptools',
|
||
dest='unzip_setuptools',
|
||
action='store_true',
|
||
help="Unzip Setuptools or Distribute when installing it")
|
||
|
||
parser.add_option(
|
||
'--relocatable',
|
||
dest='relocatable',
|
||
action='store_true',
|
||
help='Make an EXISTING virtualenv environment relocatable. '
|
||
'This fixes up scripts and makes all .pth files relative')
|
||
|
Also available in: Unified diff
Fixes #9708: Create a relay API for shared-files