+ echo '====== automated-ec2-builds [/tmp/automated-ec2-builds] ======' ====== automated-ec2-builds [/tmp/automated-ec2-builds] ====== + cd /tmp/automated-ec2-builds + bzr info Standalone tree (format: 2a) Location: branch root: . Related branches: parent branch: http://bazaar.launchpad.net/~ubuntu-on-ec2/vmbuilder/automated-ec2-builds/ + bzr version-info revision-id: ben.howard@canonical.com-20121025122633-fq8n2rbbx3xyjbq3 date: 2012-10-25 14:26:33 +0200 build-date: 2012-10-26 07:34:15 +0000 revno: 523 branch-nick: automated-ec2-builds + bzr log -p -r-1 ------------------------------------------------------------ revno: 523 committer: Ben Howard branch nick: automated-ec2-builds timestamp: Thu 2012-10-25 14:26:33 +0200 message: Finishing enablement of Raring in the build system diff: === modified file 'README.new-release' --- README.new-release 2012-04-27 21:39:39 +0000 +++ README.new-release 2012-10-25 12:26:33 +0000 @@ -1,3 +1,13 @@ +- Ubuntu 13.04 (Raring Ringtail) enablement + - No longer using crontab; build system has been re-implemented within + Jenkins + - copy conf/N-1-desktop.conf to conf/N-desktop.conf (bzr add) + - copy conf/N-1-server.conf to conf/N-server.conf (bzr add) + - add entries to daily-build-cleaner-schedule.txt + - add entries to remove-old-dailies + - copy ovf/conf/N-1-server.conf to ovf/conf/N-server.conf (bzr add) + + - Ubuntu 12.10 (Quantal Quetzal) enablement - edit crontab in automated-ec2-builds - copy conf/N-1-desktop.conf to conf/N-desktop.conf (bzr add) === modified file 'daily-build-cleaner-schedule.txt' --- daily-build-cleaner-schedule.txt 2012-08-27 21:22:41 +0000 +++ daily-build-cleaner-schedule.txt 2012-10-25 12:26:33 +0000 @@ -43,6 +43,13 @@ 5 30 all ^099720109477/ubuntu/images-testing/ebs/ubuntu-quantal-daily-%{a}-server 5 30 all ^099720109477/ubuntu/images-testing/hvm/ubuntu-quantal-daily-%{a}-desktop 5 30 all ^099720109477/ubuntu/images-testing/hvm/ubuntu-quantal-daily-%{a}-server +5 30 all ^099720109477/ubuntu/images-testing/ubuntu-raring-daily-%{a}-server +5 30 all ^099720109477/ubuntu/images-testing/ubuntu-raring-daily-%{a}-desktop +5 30 all ^099720109477/ubuntu/images-testing/ebs/ubuntu-raring-daily-%{a}-desktop +5 30 all ^099720109477/ubuntu/images-testing/ebs/ubuntu-raring-daily-%{a}-server +5 30 all ^099720109477/ubuntu/images-testing/hvm/ubuntu-raring-daily-%{a}-desktop +5 30 all ^099720109477/ubuntu/images-testing/hvm/ubuntu-raring-daily-%{a}-server + # test images 5 30 all ^099720109477/ubuntu/images-testing-dev/ubuntu-hardy-daily-%{a}-server @@ -81,6 +88,12 @@ 5 30 all ^099720109477/ubuntu/images-testing-dev/ebs/ubuntu-quantal-daily-%{a}-server 5 30 all ^099720109477/ubuntu/images-testing-dev/hvm/ubuntu-quantal-daily-%{a}-desktop 5 30 all ^099720109477/ubuntu/images-testing-dev/hvm/ubuntu-quantal-daily-%{a}-server +5 30 all ^099720109477/ubuntu/images-testing-dev/ubuntu-raring-daily-%{a}-server +5 30 all ^099720109477/ubuntu/images-testing-dev/ubuntu-raring-daily-%{a}-desktop +5 30 all ^099720109477/ubuntu/images-testing-dev/ebs/ubuntu-raring-daily-%{a}-desktop +5 30 all ^099720109477/ubuntu/images-testing-dev/ebs/ubuntu-raring-daily-%{a}-server +5 30 all ^099720109477/ubuntu/images-testing-dev/hvm/ubuntu-raring-daily-%{a}-desktop +5 30 all ^099720109477/ubuntu/images-testing-dev/hvm/ubuntu-raring-daily-%{a}-server # Rescue images 5 30 all ^099720109477/ubuntu/images-testing-dev/ubuntu-quantal-daily-%{a}-server === modified file 'remove-old-dailies' --- remove-old-dailies 2012-04-27 21:39:39 +0000 +++ remove-old-dailies 2012-10-25 12:26:33 +0000 @@ -7,7 +7,7 @@ SAFE_FILE=.save BASES=( /srv/ec2-images/server /srv/ec2-images/desktop ) -RELS="hardy intrepid jaunty karmic lucid maverick natty oneiric precise quantal" +RELS="hardy intrepid jaunty karmic lucid maverick natty oneiric precise quantal raring" # keep ${SAVE_COUNT} builds or builds to ${SAVE_DAYS} days ago, whichever is greater. export LANG=C + echo '' + echo '====== ec2-publishing-scripts [/tmp/ec2-publishing-scripts] ======' ====== ec2-publishing-scripts [/tmp/ec2-publishing-scripts] ====== + cd /tmp/ec2-publishing-scripts + bzr info Standalone tree (format: unnamed) Location: branch root: . Related branches: parent branch: http://bazaar.launchpad.net/~ubuntu-on-ec2/ubuntu-on-ec2/ec2-publishing-scripts/ + bzr version-info revision-id: ben.howard@canonical.com-20121023145341-364g93vecr6dxujd date: 2012-10-23 08:53:41 -0600 build-date: 2012-10-26 07:34:16 +0000 revno: 485 branch-nick: ec2-publishing-scripts + bzr log -p -r-1 ------------------------------------------------------------ revno: 485 committer: Ben Howard branch nick: ec2-publishing-scripts timestamp: Tue 2012-10-23 08:53:41 -0600 message: Removing Query2 as completely deprecated. diff: === modified file 'publicize-build' --- publicize-build 2012-08-28 20:24:00 +0000 +++ publicize-build 2012-10-23 14:53:41 +0000 @@ -182,15 +182,6 @@ error "updated ${url_outfile}" fi - if [ "${label}" = "daily" -o "${label}" = "release" ]; then - record-build-query2 "${allow_existing}" \ - --reg "${curfile}" \ - --files "${build_d}" \ - --build "${url_curfile}" \ - --outdir "${PUBLISH_BASE}/query2" \ - --db "/srv/builder/query2/query2.db" - fi - } short_opts="d:h" === removed file 'query-exec' --- query-exec 2012-05-02 15:08:35 +0000 +++ query-exec 1970-01-01 00:00:00 +0000 @@ -1,928 +0,0 @@ -#!/usr/bin/python -# vi: ts=4 noexpandtab - -## Copyright (C) 2011 Ben Howard -## Date: 25 October 2011 -## -## This comes with ABSOLUTELY NO WARRANTY; for details see COPYING. -## This is free software, and you are welcome to redistribute it -## under certain conditions; see copying for details. - -import os.path -import sqlite3 -import argparse -import queries -import logging -import sys -import json -import time -from datetime import date - -class ExecQuery(): - - def __init__( self, db_name, output_format, print_queries=False, opts=None): - self.db = sqlite3.connect( db_name ) - self.output = output_format - self.print_queries = print_queries - self.opts = opts - - - def exec_query(self, *args): - c = self.db.cursor() - c.execute('''PRAGMA foreign_keys = ON''') - - for query in args: - q = self.opt_replacer(query) - if self.print_queries: - print q - c.execute( q ) - - c.close() - self.db.commit() - - - def return_query(self, q): - - query = self.opt_replacer( q ) - if self.print_queries: - print query - - c = self.db.cursor() - c.execute('''PRAGMA foreign_keys = ON''') - c.execute( query ) - - columns_names = [] - returns = [] - - for desc in c.description: - columns_names.append(desc[0]) - - for row in c.fetchall(): - anon = {} - count = 0 - for tup in row: - anon[ columns_names[count] ] = tup.replace('None', 'null') - count += 1 - - returns.append(anon) - - out = returns - if self.output == 'json': - out = json.dumps( returns, sort_keys=True, indent=4) - - c.close() - return out - - def sha1( fname, sha="sha1" ): - - sha = None - - if sha == "sha1": - sha = hashlib.sha1() - else: - sha = hashlib.sha512() - - with open( fname,'rb') as f: - for chunk in iter(lambda: f.read(8192), ''): - sha.update(chunk) - - return sha.hexdigest() - - def map_distro_version_to_name(self, distro_version ): - c = self.db.cursor() - c.execute('''PRAGMA foreign_keys = ON''') - c.execute( queries.query_get_distro_name % { 'distro_version': distro_version } ) - - for row in c.fetchall(): - return row[0] - - def opt_replacer(self, string): - """ - Do variable replacement in a single place to simplify SQL mangling - """ - - o = self.opts - - if not o: - return None - - return string % { - 'region': o.region, - 'build_id': o.build_id, 'serial': o.build_id, - 'code_name': o.distro, - 'distro_name': o.distro, - 'distro_full_name': o.distro_full_name, - 'distro_version': o.distro_version, - 'distro_type': o.distro_type, - 'release_tag': o.release_tag, - 'date': o.date, - 'cloud': o.cloud, 'cloud_name': o.cloud, - 'arch': o.arch, 'arch_name': o.arch, - 'instance_type': o.instance_type, - 'release_date': o.release_date, - 'end_date': o.end_date, - 'stream': o.stream, 'stream_name': o.stream, - 'reg_id': o.reg_id, - 'distro_type': o.distro_type, - 'url': o.url, - 'public': o.public, - 'supported': o.supported, - 'description': o.description, - 'sha1': o.sha1, - 'sha512': o.sha512, - 'file_type': o.file_type, - 'file_name': o.file_name, - 'output': o.output, - 'kernel_id': o.kernel_id, - 'ramdisk_id': o.ramdisk_id, - 'register_name': o.register_name, - 'name_base': o.name_base, - 'today': date.today() - } - - '''Insert Functions''' - - def insert_region(self): - self.exec_query(queries.insert_region) - - def insert_cloud(self): - self.exec_query(queries.insert_cloud) - - def insert_distro(self): - self.exec_query(queries.insert_distro) - - def insert_distro_dates(self): - self.exec_query(queries.insert_distro_dates) - - def insert_release_tag(self): - self.exec_query(queries.insert_release_type) - - def insert_new_build(self): - - if self.opts.allow_existing: - try: - self.exec_query(queries.insert_new_build_existing) - except sqlite3.IntegrityError as e: - logger.debug("Existing row already exists...skipping ahead") - else: - self.exec_query(queries.insert_new_build) - - def insert_published(self): - - if self.opts.allow_existing: - self.exec_query(queries.insert_new_published_existing) - else: - self.exec_query(queries.insert_new_published) - - def insert_files(self): - - if self.opts.allow_existing: - self.exec_query(queries.insert_new_files_existing) - else: - self.exec_query(queries.insert_new_files) - - def insert_build_id(self): - - if self.opts.allow_existing: - try: - self.exec_query(queries.query_insert_serial_existing) - except sqlite3.IntegrityError as e: - logger.debug("Existing row already exists...skipping ahead") - else: - self.exec_query(queries.query_insert_serial) - - def update_published_public(self): - self.exec_query(queries.update_publish_public) - - def update_remove_public(self): - self.exec_query(queries.update_remove_public) - - def update_distro_end_date(self): - self.exec_query(queries.update_distro_end_date) - - def update_current_milestone(self): - self.exec_query( queries.query_update_current, queries.query_expire_serial_milestones) - - def update_activate_daily(self): - self.exec_query( queries.query_update_current, queries.query_expire_dailies_keep_three) - - def get_active_builds_for_cloud(self): - return self.return_query(queries.query_active_builds_for_cloud) - - def get_active_builds_for_cloud_distro(self): - return self.return_query(queries.query_active_builds_for_cloud_distro) - - def get_active_builds_for_cloud_distro_version(self): - return self.return_query(queries.query_active_builds_for_cloud_distro_version) - - def get_active_builds_for_region(self): - return self.return_query(queries.query_active_builds_for_region) - - def get_active_builds_for_region_distro(self): - return self.return_query(queries.query_active_builds_for_cloud_distro) - - def get_active_builds_for_cloud_arch_distro(self): - return self.return_query(queries.query_active_builds_for_cloud_arch_distro) - - def get_active_builds_for_region_arch_distro(self): - return self.return_query(queries.query_active_builds_for_region_arch_distro) - - def get_active_builds_cloud_distro_version(self): - return self.return_query(queries.query_active_builds_for_cloud_distro_version) - - def get_active_builds_cloud_distro_version_stream(self): - return self.return_query(queries.query_active_builds_for_cloud_distro_version_stream) - - def get_active_builds_cloud_distro(self): - return self.return_query(queries.query_active_builds_for_cloud_distro) - - def get_active_builds_cloud_distro_stream(self): - return self.return_query(queries.query_active_builds_for_cloud_distro_stream) - - def get_active_builds_for_cloud_arch(self): - return self.return_query(queries.query_active_builds_arches) - - def get_active_builds_arches_region(self): - return self.return_query(queries.query_active_builds_for_region_arch) - - def get_active_builds_cloud_stream(self): - return self.return_query(queries.query_get_builds_cloud_stream) - - def get_files_for_build_id(self): - return self.return_query(queries.query_files_for_build_id) - - def get_files_for_build_date(self): - return self.return_query(queries.query_files_for_build_date) - - def get_supported_distros(self): - return self.return_query(queries.query_get_supported_distros) - - def get_clouds(self): - return self.return_query(queries.query_get_clouds) - - def get_cloud(self): - return self.return_query(queries.query_get_single_cloud) - - def get_all_regions(self): - return self.return_query(queries.query_get_cloud_regions) - - def get_cloud_regions(self): - return self.return_query(queries.query_get_cloud_regions) - - def get_cloud_regions_only(self): - return self.return_query(queries.query_get_cloud_regions_only) - - def get_release_tags(self): - return self.return_query(queries.query_get_release_tags) - - def get_active_files_cloud(self): - return self.return_query(queries.query_get_files_all_active_cloud) - - def get_active_files_cloud_distro(self): - return self.return_query(queries.query_get_files_all_active_cloud_distro) - - def get_active_files_cloud_distro_version(self): - return self.return_query(queries.query_get_files_all_active_cloud_distro_version) - - def get_active_files_cloud_distro_stream(self): - return self.return_query(queries.query_get_files_all_active_cloud_distro_stream) - - def get_active_files_cloud_distro_version_stream(self): - return self.return_query(queries.query_get_files_all_active_cloud_distro_version_stream) - - def get_active_files_cloud_stream(self): - return self.return_query(queries.query_get_files_all_active_cloud_stream) - - def get_files_all_stream_distro(self): - return self.return_query(queries.query_get_files_build_stream_distro) - - def get_file_all_stream(self): - return self.return_query(queries.query_get_files_build_stream) - - def get_files_all_distro(self): - return self.return_query(queries.query_get_files_build_distro) - - def get_files_all(self): - return self.return_query(queries.query_get_files_builds_base) - - def get_files_all_distro_daily(self): - return self.return_query(queries.query_get_files_all_distro_daily) - - def get_files_all_distro_release(self): - return self.return_query(queries.query_get_files_all_distro_release) - - def get_files_all_stream_release(self): - return self.return_query(queries.query_get_files_build_stream_release) - - def get_files_all_stream_distro_stream_release(self): - return self.return_query(queries.query_get_files_build_stream_distro_release) - - def generic_query(self, query_name): - return self.return_query(getattr(queries, str("query_%s" % query_name))) - -def get_build_files(db, opts): - """ - Returns a tuple of builds and files. - """ - files = None - builds = None - - if opts.build_only: - - if opts.dailies_only: - if opts.stream and opts.distro: - files = db.generic_query("get_files_build_stream_distro_daily") - - elif opts.stream: - files = db.generic_query("get_files_all_stream_daily") - - elif opts.distro: - files = db.generic_query("get_files_all_distro_daily") - - else: - files = db.generic_query("get_files_builds_base") - - elif opts.releases_only: - if opts.stream and opts.distro: - files = db.generic_query("get_files_build_stream_distro_release") - - elif opts.stream: - files = db.generic_query("get_files_build_stream_release") - - elif opts.distro: - files = db.generic_query("get_files_all_distro_release") - - else: - files = db.generic_query("get_files_all_release") - - else: - - if opts.stream and opts.distro: - files = db.get_files_all_stream_distro() - - elif opts.stream: - files = db.get_file_all_stream() - - elif opts.distro: - files = db.get_files_all_distro() - - else: - files = db.get_files_all() - - else: - - if opts.stream and opts.distro: - - if opts.dailies_only: - builds = db.generic_query("active_builds_for_cloud_distro_stream_daily") - files = db.generic_query("get_files_all_active_cloud_distro_stream_daily") - - elif opts.releases_only: - builds = db.generic_query("active_builds_for_cloud_distro_stream_release") - files = db.generic_query("get_files_all_active_cloud_distro_stream_release") - - else: - builds = db.generic_query("active_builds_for_cloud_distro_stream") - files = db.generic_query("get_files_all_active_cloud_distro_stream") - - elif opts.stream: - - if opts.dailies_only: - files = db.generic_query("get_files_all_active_cloud_stream_daily") - builds = db.generic_query("active_builds_for_cloud_stream") - - elif opts.releases_only: - files = db.generic_query("get_files_all_active_cloud_stream_release") - builds = db.generic_query("active_builds_for_cloud_stream") - - else: - files = db.generic_query("get_files_all_active_cloud_stream") - builds = db.generic_query("active_builds_for_cloud_stream") - - elif opts.distro: - - - if opts.dailies_only: - files = db.generic_query("get_files_all_active_cloud_distro_daily") - builds = db.generic_query("active_builds_for_cloud_distro_daily") - - elif opts.releases_only: - files = db.generic_query("get_files_all_active_cloud_distro_release") - builds = db.generic_query("active_builds_for_cloud_distro_release") - - else: - files = db.generic_query("get_files_all_active_cloud_distro") - builds = db.generic_query("active_builds_for_cloud_distro") - - else: - - if opts.dailies_only: - files = db.generic_query("get_files_all_active_cloud_daily") - builds = db.generic_query("active_builds_for_cloud_daily") - - elif opts.releases_only: - files = db.generic_query("get_files_all_active_cloud_release") - builds = db.generic_query("active_builds_for_cloud_release") - - else: - files = db.generic_query("get_files_all_active_cloud") - builds = db.generic_query("active_builds_for_cloud") - - - return builds, files - -'''Yeah, this sucks, but hey, JSON is a tad bit verbose''' -def generate_json( opts ): - - json_out = {} - - vendor = {} - vendor['name'] = 'Ubuntu Cloud Images' - vendor['url'] = 'http://cloud-images.ubuntu.com' - vendor['support_url'] = 'http://www.ubuntu.com/cloud' - vendor['company'] = 'Canonical Group, Ltd' - - mirrors = {} - mirrors['authoritative'] = "https://cloud-images.ubuntu.com" - mirrors['transfer'] = [ "http://cloud-images.ubuntu.com" ] - json_out['mirrors'] = mirrors - - manifest = {} - manifest['date'] = "%s" % date.today() - manifest['serial'] = "%s" % str(time.time()).split('.')[0] - manifest['subject'] = 'Ubuntu Cloud Images' - manifest['description'] = "Active Ubuntu builds with URL and meta-data" - - db = ExecQuery( opts.db, 'raw', opts.printq, opts=opts ) - - opts.cloud = opts.cloud_json - - json_out['manifest'] = manifest - json_out['vendor'] = vendor - - builds, files = get_build_files(db, opts) - - cloud = db.get_cloud() - - for cld in cloud: - if cld['name'] == "EC2": - cld['account'] = '099720109477' - - build_catalog = [] - - for f in files: - - added_serial = False - added_bc = False - - for bc in build_catalog: - if f['distro_version'] == bc['distro_version']: - added_bc = True - - if not added_bc: - anon = {} - anon['distro_code_name'] = f['distro_code_name'] - anon['distro_version'] = f['distro_version'] - anon['build_types'] = {} - build_catalog.append( anon ) - - for bt in build_catalog: - if f['distro_version'] == bt['distro_version']: - if f['build_type'] not in bt['build_types']: - bt['build_types'][ f['build_type'] ] = [] - - added_serial = False - for nbd in bt['build_types'][ f['build_type'] ]: - if f['build_serial_id'] == nbd['build_serial_id']: - added_serial = True - - if not added_serial: - anon = {} - anon['build_serial'] = f['build_serial'] - anon['build_serial_id'] = f['build_serial_id'] - anon['release_tag'] = f['release_tag'] - anon['arches'] = {} - bt['build_types'][ f['build_type'] ].append( anon ) - - for nbd in bt['build_types'][ f['build_type'] ]: - - if nbd['build_serial_id'] == f['build_serial_id']: - - added_sub = False - for anbd in nbd['arches']: - if anbd == f['arch']: - added_sub = True - - if not added_sub: - anon = {} - anon['build_id'] = f['build_id'] - anon['file_list'] = [] - - # get cloud registrations stuff out - cloud_reg = [] - - if builds: - for b in builds: - - if b['build_id'] != f['build_id']: - continue - - c_anon = None - for c_reg in cloud_reg: - if b['cloud_name'] in c_reg['name']: - c_anon = c_reg - - if not c_anon: - c_anon = {} - c_anon['name'] = b['cloud_name'] - c_anon['instance_types'] = [] - cloud_reg.append( c_anon ) - - c_inst = False - for inst in c_anon['instance_types']: - if inst['name'] == b['instance_type']: - c_inst = inst - - if not c_inst: - c_inst = {} - c_inst['name'] = b['instance_type'] - c_inst['virt_type'] = b['virt_type'] - c_inst['registered_name'] = b['registered_name'] - c_inst['registrations'] = [] - c_anon['instance_types'].append( c_inst ) - - r = {} - r['region_name'] = b['region_name'] - r['published_id'] = b['published_id'] - r['kernel_id'] = b['kernel_id'] - r['ramdisk_id'] = b['ramdisk_id'] - c_inst['registrations'].append( r ) - - anon['cloud_registrations'] = cloud_reg - - nbd['arches'][ f['arch'] ] = anon - - for sb in nbd['arches']: - if sb == f['arch']: - anon = {} - anon['file_type'] = f['file_type'] - anon['description'] = f['file_description'] - anon['sha1'] = f['file_sha1'] - anon['sha512'] = f['file_sha512'] - anon['path'] = f['file_url'] - nbd['arches'][ f['arch'] ]['file_list'].append( anon ) - - json_out['clouds'] = cloud - json_out['catalog'] = build_catalog - - if not opts.ugly_json: - print json.dumps( json_out, sort_keys=True, indent=4) - else: - print json.dumps( json_out ) - - - -def check_args(logger, opts, *args): - - valid = True - for arg in args: - if not arg in opts: - raise Exception("invalid_argument_request", "Unknown argument %s in commandline list; argument not valid" % arg ) - - else: - if opts.__dict__[arg] is None: - logger.critical("Option requires --%s" % arg) - valid = False - - if not valid: - print opts - - return valid - -if __name__=="__main__": - - parser = argparse.ArgumentParser() - parser.add_argument('--db', action="store", required=True, - help="File name of the Database to operate on") - - parser.add_argument('--cloud_json', action="store", default=None, - help="Output Query2 Formatted json out") - - parser.add_argument('--build_json', dest="build_only", action="store_true", default=False, - help="Output Query2 Formated json with out cloud information") - - # Valid update operations - parser.add_argument('--update', action="store", - help="Update the database", - choices= [ "distro_end_date", "published_public", "remove_public", "current", "daily"]) - - # Valid insert operations - parser.add_argument('--insert', action="store", - help="Insert information into database", - choices= [ "distro", "dates", "build", "published", "cloud", "region", "release_tag", "files", "serial" ]) - - # Valid query operations - parser.add_argument('--query', action="store", - help="Query the database, i.e. get information out", - choices= [ "active_builds_cloud", - "active_builds_cloud_distro", - "active_builds_region", - "active_builds_region_distro", - "active_builds_cloud_arch", - "active_builds_arches_region", - "active_builds_regions_arch_distro", - "active_builds_regions_arch_distro", - "active_builds_cloud_distro_version", - "active_files_cloud", - "active_files_cloud_files", - "active_files_cloud_distro_stream", - "active_builds_for_cloud_distro", - "active_files_cloud_distro_version_stream", - "files_build_id", "files_build_date", "supported_distros", - "cloud", "clouds", "cloud_regions", "regions", "tags" ] ) - - # General flags - parser.add_argument('--allow_existing', action="store_true", default=False, - help="For republishing options, insert new and replace existing.") #Named "allow-..." for consistency with other tools - parser.add_argument('--distro', action="store", - help="Distribution name, i.e. precise") - parser.add_argument('--distro_full_name', action="store", - help="Distribution's full name, i.e. Precise Pangolin") - parser.add_argument('--distro_version', action="store", - help="Distribution's version number, i.e. 12.04") - parser.add_argument('--date', action="store", default=date.today(), - help="Date of the operation") - parser.add_argument('--cloud', action="store", - help="Name of the cloud, i.e. EC2") - parser.add_argument('--region', action="store", - help="Region name of the cloud, i.e. us-east-1") - parser.add_argument('--build_id', action="store", - help="Build ID of the build") - parser.add_argument('--arch', action="store", - help="Architecture type") - parser.add_argument('--instance_type', action="store", - help="Instance type") - parser.add_argument('--release_date', action="store", - help="The release date of distro") - parser.add_argument('--end_date', action="store", - help="The end of life date for the distro") - parser.add_argument('--stream', action="store", - help="The stream type, i.e. server, desktop") - parser.add_argument('--distro_type', action="store", - help="The type of distribution it is", - choices= [ "changes", "release" ]) - parser.add_argument('--reg_id', action="store", - help="Cloud provider registration ID") - parser.add_argument('--release_tag', action="store", - help="Build/release type") - parser.add_argument('--url', action="store", - help="Standard web URL, i.e. http://ubuntu.com") - parser.add_argument('--public', action="store_true", default=True, - help="Mark registration as public") - parser.add_argument('--private', dest='public', action="store_false", - help="Mark registration as not public") - parser.add_argument('--supported', action="store_true", default=False, - help="Mark as supported") - parser.add_argument('--description', action="store", - help="Description of element") - parser.add_argument('--file_type', action="store", - help="file type", - choices= [ "root.tar.gz", "qcow2","vmdk","tar.gz", "manifest", "kernel", "raw", "ovf" ] ) - parser.add_argument('--file_name', action="store", - help="Name of file") - parser.add_argument('--sha1', action= "store", - help="SHA1 of file") - parser.add_argument('--sha512', action= "store", - help="SHA512 of file") - parser.add_argument('--output', action="store", - help="Output format", default="json", - choices=[ "json", "raw" ] ) - parser.add_argument('--ugly_json', action="store_true", default=False, - help="Output condensed, ugly json") - parser.add_argument('--print', dest="printq", action="store_true", - help="Print queries on commandline", default=False ) - parser.add_argument('--kernel_id', action="store", - help="Kernel ID of published image") - parser.add_argument('--ramdisk_id', action="store", default=None, - help="Ramdisk ID of published image") - parser.add_argument('--register_name', action="store", default=None, - help="Registered name for image, i.e. 099720109477/ubuntu/images-testing/ebs/ubuntu") - parser.add_argument('--name_base', action="store", - help="Suggested base name of files for download") - parser.add_argument('--dailies_only', action="store_true", default=False, - help="For queries return just dailies") - parser.add_argument('--releases_only', action="store_true", default=False, - help="For queries return just releases, i.e. not dailies") - - opts = parser.parse_args() - - ### Now the fun part....check to make sure that options are valid - logger = logging.getLogger('_querydb_') - logging.basicConfig(format='%(asctime)s %(levelname)s - querydb - %(message)s') - logger.setLevel(logging.DEBUG) - - logger.info("Parsing commandline options") - - ### Cloud JSON - - if opts.cloud_json or opts.build_only: - - generate_json(opts) - sys.exit(0) - - opt_counts = 0 - if opts.update: - opt_counts += 1 - if opts.query: - opt_counts += 1 - if opts.insert: - opt_counts += 1 - - logger.info("Setting up connection to database") - db = ExecQuery( opts.db, opts.output, opts.printq, opts=opts ) - logger.info("Connected to SQLite3 Database at %s" % opts.db ) - - logger.info("%s operations received on commandline" % opt_counts ) - - if opt_counts > 1: - logger.critical("Only a single operation of --update, --insert, --query, --cloud_json is allowed") - sys.exit(1) - - elif opt_counts == 0: - logger.critical("Please support an operation command --update, --insert or --query") - sys.exit(1) - - - #### Update operations #### - if opts.update == "distro_end_date": - - if check_args( logger, opts, "end_date", "distro_version" ): - logger.info("Updating distro version %s to end of life on %s" % ( opts.distro_version, opts.end_date)) - - elif opts.update == "published_public": - - if check_args( logger, opts, "build_id" ): - logger.info("Marking Build %s as public" % opts.build_id) - db.update_published_public() - - elif opts.update == "remove_public": - - if check_args( logger, opts, "build_id" ): - logger.info("Marking Build %s as non-public with date of %s" % ( opts.build_id, opts.date )) - - elif opts.update == "current": - - if check_args( logger, opts, "build_id", "distro_version", "stream", "release_tag" ): - logger.info("Marking build %s %s %s %s current" % ( opts.release_tag, opts.stream, opts.build_id, opts.distro_version )) - db.update_current_milestone() - - elif opts.update == "daily": - - if check_args( logger, opts, "build_id", "distro_version", "stream" ): - logger.info("Marking build daily build %s active" % opts.build_id ) - db.update_activate_daily() - - #### Insert operations #### - if opts.insert == "distro": - - if check_args( logger, opts, "distro", "distro_full_name", "distro_version", "distro_type"): - logger.info("Adding distro %s" % opts.distro ) - db.insert_distro(); - - elif opts.insert == "dates": - - if check_args( logger, opts, "distro_version", "stream", "release_date", "end_date"): - db.insert_distro_dates() - - elif opts.insert == "build": - - if check_args( logger, opts, "build_id", "date", "distro_version", "stream", "arch", "release_tag", "name_base" ): - logger.info("Registering build id %s for %s %s" % ( opts.build_id, opts.stream, opts.arch )) - db.insert_new_build() - - elif opts.insert == "published": - - if check_args( logger, opts, "instance_type", "release_tag", "cloud", "region", "distro_version", "arch", "stream", - "build_id", "reg_id", "url", "kernel_id", "register_name"): - logger.info("Registering published build for public (%s) image %s ( %s ) %s %s" % ( opts.public, opts.cloud, opts.region, opts.release_tag, opts.reg_id)) - db.insert_published() - - elif opts.insert == "files": - - if check_args( logger, opts, "arch", "stream", "build_id", "distro_version", "file_type", "url", "release_tag"): - logger.info("Registering file %s with build %s %s %s %s" % ( opts.file_name, opts.distro_version, opts.build_id, opts.stream, opts.arch )) - db.insert_files() - - elif opts.insert == "serial": - - if check_args( logger, opts, "build_id", "distro_version", "stream", "release_tag" ): - logger.info("Recording build serial of %s %s %s" % ( opts.build_id, opts.distro_version, opts.stream )) - db.insert_build_id() - - elif opts.insert == "release_tag": - - if check_args( logger, opts, "release_tag", "description", "supported" ): - logger.info("Adding new release_tag of %s" % opts.description ) - db.insert_release_tag() - - elif opts.insert == "region": - - if check_args( logger, opts, "cloud", "region", "description", "url"): - logger.info("Adding new region %s to cloud %s" % ( opts.region, opts.cloud )) - db.insert_region() - - elif opts.insert == "cloud": - - if check_args( logger, opts, "cloud", "description", "url" ): - logger.info("Adding new cloud %s" % opts.cloud ) - db.insert_cloud() - - - ### Query Operations #### - if opts.query == "active_builds_cloud": - - if check_args( logger, opts, "cloud" ): - print db.get_active_builds_for_cloud() - - elif opts.query == "active_builds_cloud_distro": - - if check_args( logger, opts, "cloud", "distro" ): - print db.get_active_builds_cloud_distro() - - elif opts.query == "active_builds_cloud_distro_version": - - if check_args( logger, opts, "cloud", "distro_version" ): - print db.get_active_builds_cloud_distro_version() - - elif opts.query == "active_builds_for_cloud_distro": - - if check_args( logger, opts, "cloud", "distro" ): - print db.get_active_builds_for_cloud_distro() - - elif opts.query == "active_builds_region": - - if check_args( logger, opts, "cloud", "region" ): - print db.get_active_builds_for_region() - - elif opts.query == "active_builds_region_distro": - - if check_args( logger, opts, "cloud", "region", "distro" ): - print db.get_active_builds_for_region_distro() - - elif opts.query == "active_builds_regions_arch_distro": - - if check_args( logger, opts, "cloud", "region", "distro", "arch" ): - print db.get_active_builds_for_region_arch_distro() - - elif opts.query == "active_builds_arches_region": - - if check_args( logger, opts, "cloud", "region", "arch" ): - print db.get_active_builds_arches_region() - - elif opts.query == "active_files_cloud": - - if check_args( logger, opts, "cloud" ): - print db.get_active_files_cloud() - - elif opts.query == "active_files_cloud_files": - - if check_args( logger, opts, "cloud", "distro" ): - print db.get_active_files_cloud_distro() - - elif opts.query == "active_files_cloud_distro_stream": - - if check_args( logger, opts, "cloud", "distro", "stream" ): - print db.get_active_files_cloud_distro_stream() - - elif opts.query == "files_for_build_id": - pass - - elif opts.query == "files_for_build_date": - pass - - elif opts.query == "supported_distros": - - if check_args( logger, opts, "stream" ): - print db.get_supported_distros() - - elif opts.query == "clouds": - - print db.get_clouds() - - elif opts.query == "cloud": - - if check_args( logger, opts, "cloud" ): - print db.get_cloud() - - elif opts.query == "all_regions": - - print db.get_all_regions() - - elif opts.query == "tags": - - print db.get_release_tags() - - elif opts.query == "cloud_regions" : - - if check_args( logger, opts, "cloud" ): - print db.get_cloud_regions() === removed file 'record-build-query2' --- record-build-query2 2012-09-21 21:59:19 +0000 +++ record-build-query2 1970-01-01 00:00:00 +0000 @@ -1,344 +0,0 @@ -#!/bin/bash -# vi: ts=4 noexpandtab - -# This script is a transitional script for supporting query2 - -VERBOSITY=1 -PREFIX="${0}: " -skip_serial=0 - -set -e -error() { echo ${PREFIX:+"${PREFIX}"} "$@" 1>&2; } -fail() { - error "$@" - error "** FATAL query2 STOP **" - exit 1; -} -debug() { - local level=${1} - shift; - [ "${level:-0}" -gt "${VERBOSITY}" ] && return - error "$(date -R):" "${@}" -} -bad_usage(){ - cat < "${f_name}.gpg" 2>&1 ) >/dev/null && - debug 1 "Signed ${f_name}" || - fail "Unable to sign ${f_name}" - done -} - -bzr_record() { - if [ ! -d "${1}/.bzr" ]; then - ( cd "${1}" && bzr init && - bzr commit --quiet --unchanged -m "initial state" ) >/dev/null && - error "initialized bzr directory in ${info_dir}" || - fail "failed to initialize bzr directory in ${info_dir}" - fi - - ( cd "${1}" && - find . -type f -not \( -iname "*bz2*" -or -wholename "*.bzr/*" \) -exec bzr add {} + && - bzr commit -m "${distro} ${stream} ${release_tag} ${build_id}" ) >/dev/null && - debug 1 "Committed data to a bzr repository" || - fail "failed to commit data to bzr repository" -} - -generate_json() { - # Dump all the query info at the same time - python "${q_path:-${0%/*}}/querier.py" "${db_loc}" "${distro}" /srv/ec2-images/query2 || - debug "Failed to generate new Query2 data" - - tag="release" - tag_flag="--releases_only" - [ "${release_tag}" == "daily" ] && { tag="daily"; tag_flag="--dailies_only"; } - cld=$(echo $cloud | tr [:upper:] [:lower:]) - - mkdir -p "${out_d}/${stream}/${distro}" - mkdir -p "${out_d}/${stream}/${tag}/${distro}" - - #All build json - qexec --cloud_json "${cloud}" > "${out_d}/${cld}.json" - qexec --build_json > "${out_d}/builds.json" - - #All stream (i.e. desktop or servers) json - s_out_d="${out_d}/${stream}" - qexec --cloud_json "${cloud}" --stream "${stream}" > "${s_out_d}/${cld}.json" - qexec --build_json --stream "${stream}" > "${s_out_d}/builds.json" - - #Stream and suite - s_d_out_d="${out_d}/${stream}/${distro}" - qexec --cloud_json "${cloud}" --stream "${stream}" --distro "${distro}" > "${s_d_out_d}/${cld}.json" - qexec --build_json --stream "${stream}" --distro "${distro}" > "${s_d_out_d}/builds.json" - - #Stream and release tag - t_out_d="${out_d}/${stream}/${tag}" - qexec --cloud_json "${cloud}" "${tag_flag}" --stream "${stream}" > "${t_out_d}/${cld}.json" - qexec --build_json "${tag_flag}" --stream "${stream}" > "${t_out_d}/builds.json" - - #Stream, release tag and distro - t_s_out_d="${t_out_d}/${distro}" - qexec --cloud_json "${cloud}" "${tag_flag}" --stream "${stream}" --distro "${distro}" > "${t_s_out_d}/${cld}.json" - qexec --build_json "${tag_flag}" --stream "${stream}" --distro "${distro}" > "${t_s_out_d}/builds.json" - - - for d in ${s_out_d} ${s_d_out_d} ${out_d} ${t_out_d} ${t_s_out_d} - do - bzip2 -f -k "${d}/builds.json" - bzip2 -f -k "${d}/${cld}.json" - gpg_sign "${d}/${cld}.json" "${d}/${cld}.json" - gpg_sign "${d}/builds.json" "${d}/builds.json.bz2" - - for summer in md5sum sha1sum sha512sum - do - debug 1 "Running ${summer} on ${d}" - out_name="$(echo ${summer} | tr [:lower:] [:upper:])S" - find ${d} -type f -iname '*.json*' | xargs -I F -n1 ${summer} F | sed "s,${d}/,,g" > "${d}/${out_name}" - - gpg_sign "${d}/${out_name}" - done - done - bzr_record "${out_d}" -} - - -# 'Snapshot' the database and keep up to 7 days of copies of the -# database. This should give us sufficient copies of the DB -# in case things go south. - -debug 1 "** START query2 work **" -epoch=$(date +%s) -db_path="${db_loc%/*}" - -[ "${json_only:-0}" -eq 1 ] && { - generate_json - exit 0 -} - -[ -d "${db_path}/backups/${stream}/${distro}" ] || - mkdir -p "${db_path}/backups/${stream}/${distro}" - -cp "${db_loc}" "${db_path}/backups/${stream}/${distro}/${epoch}" || - fail "Failed to backup the database, not proceeding" - -cp "${db_loc}" "${db_path}/backups/database.latest" || - fail "Failed to preserve the latest copy of the database" - -# Now purge backups older than 7 days -find "${db_path}/backups/${stream}/${distro}" -used 7 -type f -exec rm {} +; - - -# Now start the work - -[ "${skip_serial:-0}" -eq 0 ] && insert_serial -arches="" - -dashed=$(echo "${release_tag}" | sed 's,\([^0-9]*\)\([0-9]\),\1-\2,') -case "${release_tag}" in - release) path="${stream}/releases/${distro}/release-${build_id}";; - daily) path="${stream}/${distro}/${build_id}";; - *) path="${stream}/releases/${distro}/${dashed}";; -esac - -for manifest in $( find ${files_d} -maxdepth 1 -iname "*.manifest" ) -do - arch=${manifest%.manifest}; arch="${arch##*-}"; - insert_build "${arch}" "${path}" -done - -for file in $(find ${files_d} -type f | egrep ".ovf$|.manifest$|.tar.gz$|.vmdk$|.img$|vmlinuz-virtual$" ) -do - debug 1 "Processing file ${file}" - case $file in - *amd64*) file_arch="amd64";; - *i386*) file_arch="i386";; - *armel*) file_arch="armel";; - *armhf*) file_arch="armhf";; - esac - - file_type="${file##*.}" - [[ "${file}" =~ ".tar.gz" ]] && file_type="tar.gz" - [[ "${file}" =~ "rootfs.tar.gz" ]] && file_type = "root.tar.gz" - [[ "${file}" =~ "vmlinuz" ]] && file_type="kernel" - - l_path="${path}" - [[ "${file}" =~ "unpacked" ]] && l_path="${path}/unpacked" - - if [[ "${file}" =~ ".img" ]]; then - f_type="$(file ${file} | tr [:lower:] [:upper:] )" - debug 1 "Identifying file ${file}" - debug 1 "${f_type}" - - case ${f_type} in - *QCOW*|*qcow*) file_type="qcow2";; - *VMDK*|*vmdk*) file_type="vmdk";; - esac - fi - - insert_files_d "${file_arch}" "${file_type}" "${l_path}/${file##*/}" "${file}" -done - -while read l_distro l_stream r_tag serial itype arch region ami aki virt_type -do - r_url="https://console.aws.amazon.com/ec2/home?region=${reigon}#launchAmi=${ami}" - - case "${release_tag}" in - daily) reg_name="099720109477/ubuntu/images-testing/ubuntu-${l_distro}-daily-${arch}-${l_stream}-${build_id}";; - alpha*|beta*) reg_name="099720109477/ubuntu/images-milestones/ubuntu-${l_distro}-MILESTONE-${arch}-${l_stream}-${build_id}"; - reg_name="${reg_name//MILESTONE/$release_tag}";; - release) reg_name="099720109477/ubuntu/images/ubuntu-${l_distro}-${l_distro_version}-${arch}-${l_stream}-${build_id}";; - esac - - if [ "${aki}" == "hvm" ]; then - reg_name="${reg_name//ubuntu/hvm/ubuntu}" - itype="hvm" - fi - - insert_published "${arch}" "${region}" "${itype}" "${ami}" "${r_url}" "${aki}" "${reg_name}" "${l_stream}" -done < ${reg} - -update_current -generate_json -debug 1 "** END query2 work **" + echo '' + echo '====== live-build [/tmp/live-build] ======' ====== live-build [/tmp/live-build] ====== + cd /tmp/live-build + bzr info Standalone tree (format: 1.9-rich-root) Location: branch root: . Related branches: parent branch: http://bazaar.launchpad.net/~ubuntu-on-ec2/live-build/cloud-images/ + bzr version-info revision-id: ben.howard@canonical.com-20121025025438-sjc5p8r097uwj83h date: 2012-10-25 04:54:38 +0200 build-date: 2012-10-26 07:34:16 +0000 revno: 1855 branch-nick: live-build + bzr log -p -r-1 ------------------------------------------------------------ revno: 1855 committer: Ben Howard branch nick: live-build timestamp: Thu 2012-10-25 04:54:38 +0200 message: Fix for missing required LXC tree diff: === modified file 'scripts/build/lb_binary_virtual-hdd-raw' --- scripts/build/lb_binary_virtual-hdd-raw 2012-06-11 17:24:38 +0000 +++ scripts/build/lb_binary_virtual-hdd-raw 2012-10-25 02:54:38 +0000 @@ -95,11 +95,32 @@ else case "${LB_BINARY_FILESYSTEM}" in ext2|ext3|ext4) - Echo_message "Creating virtual filesystem..." + Echo_message "Creating virtual filesystem...(created outside chroot)" dd if=/dev/zero of=binary-raw.img bs=1024k count=0 seek=${LB_HDD_SIZE} ${LB_ROOT_COMMAND} mkfs.${LB_BINARY_FILESYSTEM} -F binary-raw.img -L ${LB_HDD_FILESYSTEM_LABEL} -U ${FS_UUID} ${LB_ROOT_COMMAND} mount -o loop binary-raw.img binary-raw.tmp - rsync -aXHAS --one-file-system --sparse --exclude='/proc/*' --exclude="/sys/*" --exclude="/dev/*" chroot/* binary-raw.tmp/ 2> /dev/null + rsync -aXHAS \ + --one-file-system \ + --sparse \ + --exclude='/proc/*' \ + --exclude="/sys/*" \ + --exclude="/dev/*" \ + chroot/* binary-raw.tmp/ 2> /dev/null + + # Create bare LXC device tree + Echo_message "Populating virtual filesystem with bare dev tree" + mknod -m 666 binary-raw.tmp/null c 1 3 + mknod -m 666 binary-raw.tmp/zero c 1 5 + mknod -m 666 binary-raw.tmp/random c 1 8 + mknod -m 666 binary-raw.tmp/urandom c 1 9 + mkdir -m 755 binary-raw.tmp/pts + mkdir -m 1777 binary-raw.tmp/shm + mknod -m 666 binary-raw.tmp/tty c 5 0 + mknod -m 600 binary-raw.tmp/console c 5 1 + mknod -m 666 binary-raw.tmp/tty0 c 4 0 + mknod -m 666 binary-raw.tmp/full c 1 7 + mknod -m 600 binary-raw.tmp/initctl p + mknod -m 666 binary-raw.tmp/ptmx c 5 2 ;; esac fi + echo '' + echo '====== vmbuilder-0.11 [/tmp/vmbuilder-0.11] ======' ====== vmbuilder-0.11 [/tmp/vmbuilder-0.11] ====== + cd /tmp/vmbuilder-0.11 + bzr info Standalone tree (format: 2a) Location: branch root: . Related branches: parent branch: http://bazaar.launchpad.net/~ubuntu-on-ec2/vmbuilder/0.11a/ + bzr version-info revision-id: ben.howard@canonical.com-20120605221454-crv9cc4612f907lh date: 2012-06-05 16:14:54 -0600 build-date: 2012-10-26 07:34:16 +0000 revno: 398 branch-nick: vmbuilder-0.11 + bzr log -p -r-1 ------------------------------------------------------------ revno: 398 committer: Ben Howard branch nick: vmbuilder timestamp: Tue 2012-06-05 16:14:54 -0600 message: Fix for allow vmbuilder to run on 12.04 LTS. diff: === modified file 'VMBuilder/plugins/ubuntu/dapper.py' --- VMBuilder/plugins/ubuntu/dapper.py 2011-05-18 20:49:25 +0000 +++ VMBuilder/plugins/ubuntu/dapper.py 2012-06-05 22:14:54 +0000 @@ -157,18 +157,15 @@ self.vm.addpkg += ['openssh-server'] def mount_dev_proc(self): - run_cmd('mount', '--bind', '/dev', '%s/dev' % self.destdir) - self.vm.add_clean_cmd('umount', '%s/dev' % self.destdir, ignore_fail=True) - - run_cmd('mount', '--bind', '/dev/pts', '%s/dev/pts' % self.destdir) + run_cmd('mkdir', '-p', '%s/dev/pts' % self.destdir) + run_cmd('mount', '-t', 'devpts', 'devpts-live', '%s/dev/pts' % self.destdir) self.vm.add_clean_cmd('umount', '%s/dev/pts' % self.destdir, ignore_fail=True) - self.run_in_target('mount', '-t', 'proc', 'proc', '/proc') + run_cmd('mount', '-t', 'proc', 'proc-live', '%s/proc' % self.destdir) self.vm.add_clean_cmd('umount', '%s/proc' % self.destdir, ignore_fail=True) def unmount_dev_proc(self): run_cmd('umount', '%s/dev/pts' % self.destdir) - run_cmd('umount', '%s/dev' % self.destdir) run_cmd('sh', '-c', 'grep -q "$1" /proc/mounts || exit 0; umount "$1"', 'umount_binfmt', "%s/proc/sys/fs/binfmt_misc" % self.destdir) run_cmd('umount', '%s/proc' % self.destdir) + echo ''