sdk/java/log
tmp
sdk/cli/binstubs/
-sdk/cwl/arvados_cwl/_version.py
services/api/config/arvados-clients.yml
*#*
.DS_Store
ubuntu1204,ubuntu1404|requests|2.4.3|2|python|all
ubuntu1204,centos7|contextlib2|0.5.4|2|python|all
ubuntu1204,centos7|isodate|0.5.4|2|python|all
-centos7|daemon|2.1.1|2|python|all
+centos7|python-daemon|2.1.2|1|python|all
centos7|pbr|0.11.1|2|python|all
centos7|pyparsing|2.1.10|2|python|all
centos7|keepalive|0.5|2|python|all
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["scl", "enable", "python33", "/usr/local/rvm/bin/rvm-exec default bash /jenkins/run-build-packages.sh --target centos7"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian8"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian9"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1404"]
# Old versions of setuptools cannot build a schema-salad package.
RUN pip install --upgrade setuptools
+RUN git clone --depth 1 git://git.curoverse.com/arvados.git /tmp/arvados && cd /tmp/arvados/services/api && /usr/local/rvm/bin/rvm-exec default bundle && cd /tmp/arvados/apps/workbench && /usr/local/rvm/bin/rvm-exec default bundle && rm -rf /tmp/arvados
+
ENV WORKSPACE /arvados
CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu1604"]
Run package install tests
--debug
Output debug information (default: false)
+--build-version <string>
+ Version to build (default: \$ARVADOS_BUILDING_VERSION or 0.1.timestamp.commithash)
WORKSPACE=path Path to the Arvados source tree to build packages from
set -e
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,test-packages,debug,command:,only-test: \
+ help,test-packages,debug,command:,only-test:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--only-test)
ONLY_TEST="$1 $2"; shift
;;
+ --build-version)
+ ARVADOS_BUILDING_VERSION="$2"; shift
+ ;;
--)
if [ $# -gt 1 ]; then
echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
FINAL_EXITCODE=0
for dockerfile_path in $(find -name Dockerfile | grep package-build-dockerfiles); do
- if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
+ if ./run-build-packages-one-target.sh --target "$(basename $(dirname "$dockerfile_path"))" --command "$COMMAND" --build-version "$ARVADOS_BUILDING_VERSION" $DEBUG $TEST_PACKAGES $ONLY_TEST ; then
true
else
FINAL_EXITCODE=$?
Build only a specific package
--only-test <package>
Test only a specific package
+--build-version <string>
+ Version to build (default: \$ARVADOS_BUILDING_VERSION or 0.1.timestamp.commithash)
WORKSPACE=path Path to the Arvados source tree to build packages from
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,debug,test-packages,target:,command:,only-test:,only-build: \
+ help,debug,test-packages,target:,command:,only-test:,only-build:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
--test-packages)
test_packages=1
;;
+ --build-version)
+ if [[ -z "$2" ]]; then
+ :
+ elif ! [[ "$2" =~ (.*)-(.*) ]]; then
+ echo >&2 "FATAL: --build-version '$2' does not include an iteration. Try '${2}-1'?"
+ exit 1
+ else
+ ARVADOS_BUILDING_VERSION="${BASH_REMATCH[1]}"
+ ARVADOS_BUILDING_ITERATION="${BASH_REMATCH[2]}"
+ fi
+ shift
+ ;;
--)
if [ $# -gt 1 ]; then
echo >&2 "$0: unrecognized argument '$2'. Try: $0 --help"
set -e
+if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "build version='$ARVADOS_BUILDING_VERSION', package iteration='$ARVADOS_BUILDING_ITERATION'"
+fi
+
if [[ -n "$test_packages" ]]; then
if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.rpm')" ]] ; then
set +e
# Build packages
if docker run --rm \
"${docker_volume_args[@]}" \
+ --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \
+ --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \
--env ARVADOS_DEBUG=$ARVADOS_DEBUG \
--env "ONLY_BUILD=$ONLY_BUILD" \
"$IMAGE" $COMMAND
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/sdk/cwl/build"
arvados_cwl_runner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/sdk/cwl/arvados_cwl_runner.egg-info/PKG-INFO)
-arvados_cwl_runner_iteration=3
+declare -a iterargs=()
+if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ arvados_cwl_runner_iteration=3
+ iterargs+=(--iteration $arvados_cwl_runner_iteration)
+else
+ arvados_cwl_runner_iteration=
+fi
test_package_presence ${PYTHON2_PKG_PREFIX}-arvados-cwl-runner "$arvados_cwl_runner_version" python "$arvados_cwl_runner_iteration"
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" --iteration $arvados_cwl_runner_iteration
+ fpm_build $WORKSPACE/sdk/cwl "${PYTHON2_PKG_PREFIX}-arvados-cwl-runner" 'Curoverse, Inc.' 'python' "$arvados_cwl_runner_version" "--url=https://arvados.org" "--description=The Arvados CWL runner" --depends "${PYTHON2_PKG_PREFIX}-setuptools" "${iterargs[@]}"
fi
# schema_salad. This is a python dependency of arvados-cwl-runner,
#
# Ward, 2016-03-17
saladversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep schema-salad== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-schema-salad "$saladversion" python
+test_package_presence python-schema-salad "$saladversion" python 2
if [[ "$?" == "0" ]]; then
- fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2"
+ fpm_build schema_salad "" "" python $saladversion --depends "${PYTHON2_PKG_PREFIX}-lockfile >= 1:0.12.2-2" --depends "${PYTHON2_PKG_PREFIX}-avro = 1.8.1-2" --iteration 2
fi
# And for cwltool we have the same problem as for schema_salad. Ward, 2016-03-17
cwltoolversion=$(cat "$WORKSPACE/sdk/cwl/setup.py" | grep cwltool== | sed "s/.*==\(.*\)'.*/\1/")
-test_package_presence python-cwltool "$cwltoolversion" python
+test_package_presence python-cwltool "$cwltoolversion" python 2
if [[ "$?" == "0" ]]; then
- fpm_build cwltool "" "" python $cwltoolversion
+ fpm_build cwltool "" "" python $cwltoolversion --iteration 2
fi
# The PAM module
cd $WORKSPACE/packages/$TARGET
rm -rf "$WORKSPACE/services/dockercleaner/build"
dockercleaner_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/services/dockercleaner/arvados_docker_cleaner.egg-info/PKG-INFO)
-dockercleaner_iteration=3
-test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$dockercleaner_iteration"
+iteration="${ARVADOS_BUILDING_ITERATION:-3}"
+test_package_presence arvados-docker-cleaner "$dockercleaner_version" python "$iteration"
if [[ "$?" == "0" ]]; then
- fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$dockercleaner_iteration"
+ fpm_build $WORKSPACE/services/dockercleaner arvados-docker-cleaner 'Curoverse, Inc.' 'python3' "$dockercleaner_version" "--url=https://arvados.org" "--description=The Arvados Docker image cleaner" --depends "${PYTHON3_PKG_PREFIX}-websocket-client = 0.37.0" --iteration "$iteration"
fi
# The Arvados crunchstat-summary tool
cd $WORKSPACE/packages/$TARGET
crunchstat_summary_version=$(awk '($1 == "Version:"){print $2}' $WORKSPACE/tools/crunchstat-summary/crunchstat_summary.egg-info/PKG-INFO)
-test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python
+iteration="${ARVADOS_BUILDING_ITERATION:-2}"
+test_package_presence "$PYTHON2_PKG_PREFIX"-crunchstat-summary "$crunchstat_summary_version" python "$iteration"
if [[ "$?" == "0" ]]; then
rm -rf "$WORKSPACE/tools/crunchstat-summary/build"
- fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage"
+ fpm_build $WORKSPACE/tools/crunchstat-summary ${PYTHON2_PKG_PREFIX}-crunchstat-summary 'Curoverse, Inc.' 'python' "$crunchstat_summary_version" "--url=https://arvados.org" "--description=Crunchstat-summary reads Arvados Crunch log files and summarize resource usage" --iteration "$iteration"
fi
-if [[ -z "$ONLY_BUILD" ]] || [[ "${PYTHON2_PKG_PREFIX}-apache-libcloud" == "$ONLY_BUILD" ]] ; then
- # Forked libcloud
+# Forked libcloud
+if test_package_presence "$PYTHON2_PKG_PREFIX"-apache-libcloud "$LIBCLOUD_PIN" python 2
+then
LIBCLOUD_DIR=$(mktemp -d)
(
cd $LIBCLOUD_DIR
handle_python_package
DASHQ_UNLESS_DEBUG=$OLD_DASHQ_UNLESS_DEBUG
)
- fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud
+ fpm_build $LIBCLOUD_DIR "$PYTHON2_PKG_PREFIX"-apache-libcloud "" python "" --iteration 2
rm -rf $LIBCLOUD_DIR
fi
# We need to bundle to be ready even when we build a package without vendor directory
# because asset compilation requires it.
- bundle install --path vendor/bundle >"$STDOUT_IF_DEBUG"
+ bundle install --system >"$STDOUT_IF_DEBUG"
# clear the tmp directory; the asset generation step will recreate tmp/cache/assets,
# and we want that in the package, so it's easier to not exclude the tmp directory
# older packages.
LICENSE_PACKAGE_TS=20151208015500
-RAILS_PACKAGE_ITERATION=8
+if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ RAILS_PACKAGE_ITERATION=8
+else
+ RAILS_PACKAGE_ITERATION="$ARVADOS_BUILDING_ITERATION"
+fi
debug_echo () {
echo "$@" >"$STDOUT_IF_DEBUG"
}
version_from_git() {
- # Generates a version number from the git log for the current working
- # directory, and writes it to stdout.
- local git_ts git_hash prefix
- if [[ -n "$1" ]] ; then
- prefix="$1"
- else
- prefix="0.1"
- fi
+ # Output the version being built, or if we're building a
+ # dev/prerelease, output a version number based on the git log for
+ # the current working directory.
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "$ARVADOS_BUILDING_VERSION"
+ return
+ fi
+
+ local git_ts git_hash prefix
+ if [[ -n "$1" ]] ; then
+ prefix="$1"
+ else
+ prefix="0.1"
+ fi
- declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
- echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
+ declare $(format_last_commit_here "git_ts=%ct git_hash=%h")
+ echo "${prefix}.$(date -ud "@$git_ts" +%Y%m%d%H%M%S).$git_hash"
}
nohash_version_from_git() {
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "$ARVADOS_BUILDING_VERSION"
+ return
+ fi
version_from_git $1 | cut -d. -f1-3
}
go get -ldflags "-X main.version=${version}" "git.curoverse.com/arvados.git/$src_path"
- declare -a switches=()
+ local -a switches=()
systemd_unit="$WORKSPACE/${src_path}/${prog}.service"
if [[ -e "${systemd_unit}" ]]; then
switches+=(
}
default_iteration() {
+ if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
+ echo "$ARVADOS_BUILDING_ITERATION"
+ return
+ fi
local package_name="$1"; shift
local package_version="$1"; shift
local package_type="$1"; shift
cd $tmppwd
- test_package_presence $pkgname $version rails $RAILS_PACKAGE_ITERATION
+ test_package_presence $pkgname $version rails "$RAILS_PACKAGE_ITERATION"
}
test_package_presence() {
fi
if [[ "$iteration" == "" ]]; then
- iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
+ iteration="$(default_iteration "$pkgname" "$version" "$pkgtype")"
fi
if [[ "$arch" == "" ]]; then
fi
if [[ "$FORMAT" == "deb" ]]; then
- local complete_pkgname=$pkgname"_"$version"-"$iteration"_"$deb_architecture".deb"
+ local complete_pkgname="${pkgname}_$version${iteration:+-$iteration}_$deb_architecture.deb"
else
- local complete_pkgname="$pkgname-$version-$iteration.$rpm_architecture.rpm"
+ # rpm packages get iteration 1 if we don't supply one
+ iteration=${iteration:-1}
+ local complete_pkgname="$pkgname-$version-${iteration}.$rpm_architecture.rpm"
fi
# See if we can skip building the package, only if it already exists in the
return 0
fi
local srcdir="$1"; shift
+ cd "$srcdir"
local license_path="$1"; shift
+ local version="$(version_from_git)"
local scripts_dir="$(mktemp --tmpdir -d "$pkgname-XXXXXXXX.scripts")" && \
- local version_file="$(mktemp --tmpdir "$pkgname-XXXXXXXX.version")" && (
+ (
set -e
_build_rails_package_scripts "$pkgname" "$scripts_dir"
cd "$srcdir"
mkdir -p tmp
- version_from_git >"$version_file"
git rev-parse HEAD >git-commit.version
bundle package --all
)
if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then
echo "ERROR: $pkgname package prep failed" >&2
- rm -rf "$scripts_dir" "$version_file"
+ rm -rf "$scripts_dir"
EXITCODE=1
return 1
fi
local railsdir="/var/www/${pkgname%-server}/current"
- local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir
- "$(cat "$version_file")")
+ local -a pos_args=("$srcdir/=$railsdir" "$pkgname" "Curoverse, Inc." dir "$version")
local license_arg="$license_path=$railsdir/$(basename "$license_path")"
- local -a switches=(--iteration=$RAILS_PACKAGE_ITERATION
- --after-install "$scripts_dir/postinst"
+ local -a switches=(--after-install "$scripts_dir/postinst"
--before-remove "$scripts_dir/prerm"
--after-remove "$scripts_dir/postrm")
+ if [[ -z "$ARVADOS_BUILDING_VERSION" ]]; then
+ switches+=(--iteration $RAILS_PACKAGE_ITERATION)
+ fi
# For some reason fpm excludes need to not start with /.
local exclude_root="${railsdir#/}"
# .git and packages are for the SSO server, which is built from its
switches+=(-x "$exclude_root/$exclude")
done
fpm_build "${pos_args[@]}" "${switches[@]}" \
+ -x "$exclude_root/vendor/cache-*" \
-x "$exclude_root/vendor/bundle" "$@" "$license_arg"
- rm -rf "$scripts_dir" "$version_file"
+ rm -rf "$scripts_dir"
}
# Build packages for everything
if [[ "$VERSION" != "" ]]; then
COMMAND_ARR+=('-v' "$VERSION")
fi
- # We can always add an --iteration here. If another one is specified in $@,
- # that will take precedence, as desired.
- COMMAND_ARR+=(--iteration "$default_iteration_value")
+ if [[ -n "$default_iteration_value" ]]; then
+ # We can always add an --iteration here. If another one is specified in $@,
+ # that will take precedence, as desired.
+ COMMAND_ARR+=(--iteration "$default_iteration_value")
+ fi
if [[ python = "$PACKAGE_TYPE" ]] && [[ -e "${PACKAGE}/${PACKAGE_NAME}.service" ]]
then
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pkg_resources
+
+__version__ = pkg_resources.require('arvados-cwl-runner')[0].version
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
-
-versionfile = os.path.join(SETUP_DIR, "arvados_cwl/_version.py")
-try:
- gitinfo = subprocess.check_output(
- ['git', 'log', '--first-parent', '--max-count=1',
- '--format=format:%H', gittaggers.choose_version_from()]).strip()
- with open(versionfile, "w") as f:
- f.write("__version__ = '%s'\n" % gitinfo)
-except Exception as e:
- # When installing from package, it won't be part of a git repository, and
- # check_output() will raise an exception. But the package should include the
- # version file, so we can proceed.
- if not os.path.exists(versionfile):
- raise
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "1.0"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='arvados-cwl-runner',
- version='1.0',
+ version=version,
description='Arvados Common Workflow Language runner',
long_description=open(README).read(),
author='Arvados',
README = os.path.join(SETUP_DIR, 'README.rst')
tagger = egg_info_cmd.egg_info
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except (ImportError, OSError):
- pass
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='arvados-pam',
- version='0.1',
+ version=version,
description='Arvados PAM module',
long_description=open(README).read(),
author='Arvados',
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
short_tests_only = False
if '--short-tests-only' in sys.argv:
sys.argv.remove('--short-tests-only')
setup(name='arvados-python-client',
- version='0.1',
+ version=version,
description='Arvados client library',
long_description=open(README).read(),
author='Arvados',
def self.full_text_searchable_columns
self.columns.select do |col|
- col.type == :string or col.type == :text
+ [:string, :text, :jsonb].include?(col.type)
end.map(&:name)
end
def self.full_text_tsvector
parts = full_text_searchable_columns.collect do |column|
- "coalesce(#{column},'')"
+ cast = serialized_attributes[column] ? '::text' : ''
+ "coalesce(#{column}#{cast},'')"
end
"to_tsvector('english', #{parts.join(" || ' ' || ")})"
end
}
end
+ def replace_index(t)
+ i = fts_indexes[t]
+ t.classify.constantize.reset_column_information
+ execute "DROP INDEX IF EXISTS #{i}"
+ execute "CREATE INDEX #{i} ON #{t} USING gin(#{t.classify.constantize.full_text_tsvector})"
+ end
+
def up
- # remove existing fts indexes and create up to date ones with no leading space
- fts_indexes.each do |t, i|
- t.classify.constantize.reset_column_information
- ActiveRecord::Base.connection.indexes(t).each do |idx|
- if idx.name == i
- remove_index t.to_sym, :name => i
- break
- end
- end
- execute "CREATE INDEX #{i} ON #{t} USING gin(#{t.classify.constantize.full_text_tsvector});"
+ # remove existing fts indexes and create up to date ones with no
+ # leading space
+ fts_indexes.keys.each do |t|
+ replace_index(t)
end
end
--- /dev/null
+require 'migrate_yaml_to_json'
+
+class YamlToJson < ActiveRecord::Migration
+ def up
+ [
+ ['collections', 'properties'],
+ ['containers', 'environment'],
+ ['containers', 'mounts'],
+ ['containers', 'runtime_constraints'],
+ ['containers', 'command'],
+ ['containers', 'scheduling_parameters'],
+ ['container_requests', 'properties'],
+ ['container_requests', 'environment'],
+ ['container_requests', 'mounts'],
+ ['container_requests', 'runtime_constraints'],
+ ['container_requests', 'command'],
+ ['container_requests', 'scheduling_parameters'],
+ ['humans', 'properties'],
+ ['job_tasks', 'parameters'],
+ ['links', 'properties'],
+ ['nodes', 'info'],
+ ['nodes', 'properties'],
+ ['pipeline_instances', 'components'],
+ ['pipeline_instances', 'properties'],
+ ['pipeline_instances', 'components_summary'],
+ ['pipeline_templates', 'components'],
+ ['specimens', 'properties'],
+ ['traits', 'properties'],
+ ['users', 'prefs'],
+ ].each do |table, column|
+ MigrateYAMLToJSON.migrate(table, column)
+ end
+ end
+
+ def down
+ end
+end
--- /dev/null
+require './db/migrate/20161213172944_full_text_search_indexes'
+
+class JsonCollectionProperties < ActiveRecord::Migration
+ def up
+ # Drop the FT index before changing column type to avoid
+ # "PG::DatatypeMismatch: ERROR: COALESCE types jsonb and text
+ # cannot be matched".
+ ActiveRecord::Base.connection.execute 'DROP INDEX IF EXISTS collections_full_text_search_idx'
+ ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN properties TYPE jsonb USING properties::jsonb'
+ FullTextSearchIndexes.new.replace_index('collections')
+ end
+
+ def down
+ ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN properties TYPE text'
+ end
+end
--- /dev/null
+class FixTrashFlagFollow < ActiveRecord::Migration
+ def change
+ ActiveRecord::Base.connection.execute("DROP MATERIALIZED VIEW materialized_permission_view")
+ ActiveRecord::Base.connection.execute(
+"-- constructing perm_edges
+-- 1. get the list of all permission links,
+-- 2. any can_manage link or permission link to a group means permission should 'follow through'
+-- (as a special case, can_manage links to a user grant access to everything owned by the user,
+-- unlike can_read or can_write which only grant access to the user record)
+-- 3. add all owner->owned relationships between groups as can_manage edges
+--
+-- constructing permissions
+-- 1. base case: start with set of all users as the working set
+-- 2. recursive case:
+-- join with edges where the tail is in the working set and 'follow' is true
+-- produce a new working set with the head (target) of each edge
+-- set permission to the least permission encountered on the path
+-- propagate trashed flag down
+
+CREATE MATERIALIZED VIEW materialized_permission_view AS
+WITH RECURSIVE
+perm_value (name, val) AS (
+ VALUES
+ ('can_read', 1::smallint),
+ ('can_login', 1),
+ ('can_write', 2),
+ ('can_manage', 3)
+ ),
+perm_edges (tail_uuid, head_uuid, val, follow, trashed) AS (
+ SELECT links.tail_uuid,
+ links.head_uuid,
+ pv.val,
+ (pv.val = 3 OR groups.uuid IS NOT NULL) AS follow,
+ 0::smallint AS trashed,
+ 0::smallint AS followtrash
+ FROM links
+ LEFT JOIN perm_value pv ON pv.name = links.name
+ LEFT JOIN groups ON pv.val<3 AND groups.uuid = links.head_uuid
+ WHERE links.link_class = 'permission'
+ UNION ALL
+ SELECT owner_uuid, uuid, 3, true,
+ CASE WHEN trash_at IS NOT NULL and trash_at < clock_timestamp() THEN 1 ELSE 0 END,
+ 1
+ FROM groups
+ ),
+perm (val, follow, user_uuid, target_uuid, trashed) AS (
+ SELECT 3::smallint AS val,
+ true AS follow,
+ users.uuid::varchar(32) AS user_uuid,
+ users.uuid::varchar(32) AS target_uuid,
+ 0::smallint AS trashed
+ FROM users
+ UNION
+ SELECT LEAST(perm.val, edges.val)::smallint AS val,
+ edges.follow AS follow,
+ perm.user_uuid::varchar(32) AS user_uuid,
+ edges.head_uuid::varchar(32) AS target_uuid,
+ (GREATEST(perm.trashed, edges.trashed) * edges.followtrash)::smallint AS trashed
+ FROM perm
+ INNER JOIN perm_edges edges
+ ON perm.follow AND edges.tail_uuid = perm.target_uuid
+)
+SELECT user_uuid,
+ target_uuid,
+ MAX(val) AS perm_level,
+ CASE follow WHEN true THEN target_uuid ELSE NULL END AS target_owner_uuid,
+ MAX(trashed) AS trashed
+ FROM perm
+ GROUP BY user_uuid, target_uuid, target_owner_uuid;
+")
+ add_index :materialized_permission_view, [:trashed, :target_uuid], name: 'permission_target_trashed'
+ add_index :materialized_permission_view, [:user_uuid, :trashed, :perm_level], name: 'permission_target_user_trashed_level'
+ ActiveRecord::Base.connection.execute("REFRESH MATERIALIZED VIEW materialized_permission_view")
+ end
+end
manifest_text text,
name character varying(255),
description character varying(524288),
- properties text,
+ properties jsonb,
delete_at timestamp without time zone,
file_names character varying(8192),
trash_at timestamp without time zone,
links.head_uuid,
pv.val,
((pv.val = 3) OR (groups.uuid IS NOT NULL)) AS follow,
- (0)::smallint AS trashed
+ (0)::smallint AS trashed,
+ (0)::smallint AS followtrash
FROM ((links
LEFT JOIN perm_value pv ON ((pv.name = (links.name)::text)))
LEFT JOIN groups ON (((pv.val < 3) AND ((groups.uuid)::text = (links.head_uuid)::text))))
CASE
WHEN ((groups.trash_at IS NOT NULL) AND (groups.trash_at < clock_timestamp())) THEN 1
ELSE 0
- END AS "case"
+ END AS "case",
+ 1
FROM groups
), perm(val, follow, user_uuid, target_uuid, trashed) AS (
SELECT (3)::smallint AS val,
edges.follow,
perm_1.user_uuid,
(edges.head_uuid)::character varying(32) AS target_uuid,
- (GREATEST((perm_1.trashed)::integer, edges.trashed))::smallint AS trashed
+ ((GREATEST((perm_1.trashed)::integer, edges.trashed) * edges.followtrash))::smallint AS trashed
FROM (perm perm_1
JOIN perm_edges edges ON ((perm_1.follow AND ((edges.tail_uuid)::text = (perm_1.target_uuid)::text))))
)
-- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
+CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
--
INSERT INTO schema_migrations (version) VALUES ('20170628185847');
+INSERT INTO schema_migrations (version) VALUES ('20170704160233');
+
+INSERT INTO schema_migrations (version) VALUES ('20170706141334');
+
INSERT INTO schema_migrations (version) VALUES ('20170824202826');
INSERT INTO schema_migrations (version) VALUES ('20170906224040');
INSERT INTO schema_migrations (version) VALUES ('20171027183824');
+INSERT INTO schema_migrations (version) VALUES ('20171208203841');
end
def self.load(s)
- if s.nil?
+ if s.is_a?(object_class)
+ # Rails already deserialized for us
+ s
+ elsif s.nil?
object_class.new()
elsif s[0] == first_json_char
SafeJSON.load(s)
assert_equal true, (updated_at_2 > updated_at_1), "Expected updated time 2 to be newer than 1"
end
+
+ test 'jsonb column' do
+ set_user_from_auth :active
+
+ c = Collection.create!(properties: {})
+ assert_equal({}, c.properties)
+
+ c.update_attributes(properties: {'foo' => 'foo'})
+ c.reload
+ assert_equal({'foo' => 'foo'}, c.properties)
+
+ c.update_attributes(properties: nil)
+ c.reload
+ assert_equal({}, c.properties)
+
+ c.update_attributes(properties: {foo: 'bar'})
+ assert_equal({'foo' => 'bar'}, c.properties)
+ c.reload
+ assert_equal({'foo' => 'bar'}, c.properties)
+ end
end
assert Group.readable_by(users(:active)).where(uuid: g_baz.uuid).any?
end
+ test "trashed does not propagate across permission links" do
+ set_user_from_auth :admin
+
+ g_foo = Group.create!(name: "foo")
+ u_bar = User.create!(first_name: "bar")
+
+ assert Group.readable_by(users(:admin)).where(uuid: g_foo.uuid).any?
+ assert User.readable_by(users(:admin)).where(uuid: u_bar.uuid).any?
+ g_foo.update! is_trashed: true
+
+ assert Group.readable_by(users(:admin)).where(uuid: g_foo.uuid).empty?
+ assert User.readable_by(users(:admin)).where(uuid: u_bar.uuid).any?
+
+ g_foo.update! is_trashed: false
+ ln = Link.create!(tail_uuid: g_foo.uuid,
+ head_uuid: u_bar.uuid,
+ link_class: "permission",
+ name: "can_read")
+ g_foo.update! is_trashed: true
+
+ assert Group.readable_by(users(:admin)).where(uuid: g_foo.uuid).empty?
+ assert User.readable_by(users(:admin)).where(uuid: u_bar.uuid).any?
+ end
+
end
from setuptools import setup, find_packages
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name="arvados-docker-cleaner",
- version="0.1",
+ version=version,
description="Arvados Docker cleaner",
author="Arvados",
author_email="info@arvados.org",
SETUP_DIR = os.path.dirname(__file__) or '.'
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
short_tests_only = False
if '--short-tests-only' in sys.argv:
sys.argv.remove('--short-tests-only')
setup(name='arvados_fuse',
- version='0.1',
+ version=version,
description='Arvados FUSE driver',
long_description=open(README).read(),
author='Arvados',
SETUP_DIR = os.path.dirname(__file__) or "."
README = os.path.join(SETUP_DIR, 'README.rst')
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='arvados-node-manager',
- version='0.1',
+ version=version,
description='Arvados compute node manager',
long_description=open(README).read(),
author='Arvados',
yarn install
if test "$1" != "--only-deps" ; then
- echo "apiEndPoint: https://${localip}:${services[api]}" > /usr/src/composer/src/arvados-configuration.yml
+ echo "apiEndPoint: https://${localip}:${services[api]}" > /usr/src/composer/src/composer.yml
exec ng serve --host 0.0.0.0 --port 4200 --env=webdev
fi
SETUP_DIR = os.path.dirname(__file__) or '.'
-try:
- import gittaggers
- tagger = gittaggers.EggInfoFromGit
-except ImportError:
- tagger = egg_info_cmd.egg_info
+tagger = egg_info_cmd.egg_info
+version = os.environ.get("ARVADOS_BUILDING_VERSION")
+if not version:
+ version = "0.1"
+ try:
+ import gittaggers
+ tagger = gittaggers.EggInfoFromGit
+ except ImportError:
+ pass
setup(name='crunchstat_summary',
- version='0.1',
+ version=version,
description='read crunch log files and summarize resource usage',
author='Arvados',
author_email='info@arvados.org',
projects / arvados.git / commitdiff