diff --git a/base/cvd/cuttlefish/files/BUILD.bazel b/base/cvd/cuttlefish/files/BUILD.bazel index 78dfbd204c7..168c7a9fa67 100644 --- a/base/cvd/cuttlefish/files/BUILD.bazel +++ b/base/cvd/cuttlefish/files/BUILD.bazel @@ -29,6 +29,18 @@ cf_cc_library( ], ) +cf_cc_library( + name = "copy_with_attributes", + srcs = ["copy_with_attributes.cc"], + hdrs = ["copy_with_attributes.h"], + deps = [ + "//cuttlefish/files:copy", + "//cuttlefish/posix:strerror", + "//cuttlefish/result:expect", + "//cuttlefish/result:result_type", + ], +) + cf_cc_library( name = "directory_contents", srcs = ["directory_contents.cc"], @@ -78,7 +90,7 @@ cf_cc_library( hdrs = ["link_or_copy.h"], deps = [ "//cuttlefish/files:are_hard_linked", - "//cuttlefish/files:copy", + "//cuttlefish/files:copy_with_attributes", "//cuttlefish/files:file_exists", "//cuttlefish/posix:strerror", "//cuttlefish/result:expect", diff --git a/base/cvd/cuttlefish/files/copy_with_attributes.cc b/base/cvd/cuttlefish/files/copy_with_attributes.cc new file mode 100644 index 00000000000..4c1e2d8147c --- /dev/null +++ b/base/cvd/cuttlefish/files/copy_with_attributes.cc @@ -0,0 +1,42 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "cuttlefish/files/copy_with_attributes.h" + +#include +#include + +#include + +#include "cuttlefish/files/copy.h" +#include "cuttlefish/posix/strerror.h" +#include "cuttlefish/result/expect.h" +#include "cuttlefish/result/result_type.h" + +namespace cuttlefish { + +Result CopyWithAttributes(const std::string& from, + const std::string& to) { + CF_EXPECTF(Copy(from, to), "Failed to copy '{}' to '{}'", from, to); + struct stat st; + CF_EXPECTF(stat(from.c_str(), &st) >= 0, "Failed to stat '{}': {}", from, + StrError(errno)); + CF_EXPECTF(chmod(to.c_str(), st.st_mode) >= 0, "Failed to chmod '{}': {}", to, + StrError(errno)); + return {}; +} + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/files/copy_with_attributes.h b/base/cvd/cuttlefish/files/copy_with_attributes.h new file mode 100644 index 00000000000..b055317e7f8 --- /dev/null +++ b/base/cvd/cuttlefish/files/copy_with_attributes.h @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2017 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include + +#include "cuttlefish/result/result_type.h" + +namespace cuttlefish { + +Result CopyWithAttributes(const std::string& from, const std::string& to); + +} // namespace cuttlefish diff --git a/base/cvd/cuttlefish/files/link_or_copy.cc b/base/cvd/cuttlefish/files/link_or_copy.cc index 5e85c9fcb3d..7b582bbae58 100644 --- a/base/cvd/cuttlefish/files/link_or_copy.cc +++ b/base/cvd/cuttlefish/files/link_or_copy.cc @@ -24,7 +24,7 @@ #include "absl/log/log.h" #include "cuttlefish/files/are_hard_linked.h" -#include "cuttlefish/files/copy.h" +#include "cuttlefish/files/copy_with_attributes.h" #include "cuttlefish/files/file_exists.h" #include "cuttlefish/posix/strerror.h" #include "cuttlefish/result/expect.h" @@ -56,8 +56,7 @@ Result LinkOrCopy(const std::string& target, << destination << "\""; return destination; } - CF_EXPECTF(Copy(target, destination), "Failed to copy \"{}\" to \"{}\"", - target, destination); + CF_EXPECT(CopyWithAttributes(target, destination)); VLOG(1) << "Copied file from \"" << target << "\" to \"" << destination << "\""; diff --git a/base/cvd/cuttlefish/package/BUILD.bazel b/base/cvd/cuttlefish/package/BUILD.bazel index 389bfba4b8c..48c0b79b3b8 100644 --- a/base/cvd/cuttlefish/package/BUILD.bazel +++ b/base/cvd/cuttlefish/package/BUILD.bazel @@ -1,14 +1,33 @@ -load("//cuttlefish/package:rules.bzl", "package_files") +load("//cuttlefish/bazel:rules.bzl", "cf_cc_binary") +load("//cuttlefish/package:rules.bzl", "package_executable", "package_files") -alias( +cf_cc_binary( + name = "packager", + srcs = ["packager.cc"], + deps = [ + "//cuttlefish/common/libs/utils:files", + "//cuttlefish/files:copy_with_attributes", + "//cuttlefish/files:link_or_copy", + "//cuttlefish/files:recursively_remove_directory", + "//cuttlefish/flag_parser", + "//cuttlefish/posix:strerror", + "//cuttlefish/posix:symlink", + "//cuttlefish/result:expect", + "//cuttlefish/result:result_type", + "//libbase", + "@abseil-cpp//absl/strings", + ], +) + +package_executable( name = "cvd", - actual = ":common", + executable = "bin/cvd", + package = ":common", ) package_files( name = "common", base_dir = "cuttlefish-common", - executable = "bin/cvd", package_file_symlink_to_package_file = { "bin/mcopy": "bin/mtools", "bin/mmd": "bin/mtools", @@ -158,7 +177,6 @@ package_files( package_files( name = "defaults", base_dir = "cuttlefish-integration", - executable = "bin/cf_defaults", package_file_to_src = { "bin/cf_defaults": "//cuttlefish/host/commands/defaults:cf_defaults", }, @@ -168,7 +186,6 @@ package_files( package_files( name = "metrics", base_dir = "cuttlefish-metrics", - executable = "bin/metrics_transmitter", package_file_to_src = { "bin/metrics_transmitter": "//cuttlefish/host/commands/metrics_transmitter:metrics_transmitter", }, diff --git a/base/cvd/cuttlefish/package/packager.cc b/base/cvd/cuttlefish/package/packager.cc new file mode 100644 index 00000000000..e60213ca15f --- /dev/null +++ b/base/cvd/cuttlefish/package/packager.cc @@ -0,0 +1,163 @@ +// +// Copyright (C) 2026 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" +#include "android-base/file.h" + +#include "cuttlefish/common/libs/utils/files.h" +#include "cuttlefish/files/copy_with_attributes.h" +#include "cuttlefish/files/link_or_copy.h" +#include "cuttlefish/files/recursively_remove_directory.h" +#include "cuttlefish/flag_parser/flag.h" +#include "cuttlefish/flag_parser/gflags_compat.h" +#include "cuttlefish/posix/strerror.h" +#include "cuttlefish/posix/symlink.h" +#include "cuttlefish/result/expect.h" +#include "cuttlefish/result/result_type.h" + +namespace cuttlefish { +namespace { + +Result InsertNewPair( + std::map>& map, + std::string_view key_equals_value) { + std::pair pair = + absl::StrSplit(key_equals_value, absl::MaxSplits('=', 1)); + CF_EXPECT(!pair.first.empty()); + CF_EXPECT(!pair.second.empty()); + CF_EXPECTF(map.emplace(pair.first, pair.second).second, + "Duplicate key '{}' in '{}'", pair.first, key_equals_value); + return {}; +} + +class Args { + public: + static Result Parse(std::vector strs) { + Args args; + ConsumeFlagsOpts opts = {.fail_on_unexpected_argument = true}; + CF_EXPECT(ConsumeFlags(args.Flags(), std::move(strs), opts)); + return args; + } + + const std::string& BaseDir() const { return base_dir_; } + + const std::map>& PackageToSrc() + const { + return package_to_src_; + } + + const std::map>& + PackageFileSymlinkToPackageFile() const { + return package_file_symlink_to_package_file_; + } + + private: + Args() = default; + + std::vector Flags() { + return { + GflagsCompatFlag("base_dir", base_dir_), + Flag::StringFlag("package_file_to_src") + .Setter(std::bind_front(InsertNewPair, std::ref(package_to_src_))), + Flag::StringFlag("package_file_symlink_to_package_file") + .Setter(std::bind_front( + InsertNewPair, + std::ref(package_file_symlink_to_package_file_))), + }; + } + + std::string base_dir_; + std::map> package_to_src_; + std::map> + package_file_symlink_to_package_file_; +}; + +std::string GetRelativePathForLink(std::string_view target_path, + std::string_view link_path) { + std::list target = absl::StrSplit(target_path, "/"); + std::list link = absl::StrSplit(link_path, "/"); + while (!target.empty() && !link.empty() && *target.begin() == *link.begin()) { + target.pop_front(); + link.pop_front(); + } + for (size_t i = 0; i < link.size() - 1; ++i) { + target.push_front(".."); + } + return absl::StrJoin(target, "/"); +} + +Result PackagerMain(std::vector args_strs) { + const Args args = CF_EXPECT(Args::Parse(std::move(args_strs))); + + CF_EXPECT(RecursivelyRemoveDirectory(args.BaseDir())); + + for (const auto& [pkg_path, src_path] : args.PackageToSrc()) { + const std::string base_pkg = absl::StrCat(args.BaseDir(), "/", pkg_path); + CF_EXPECT(EnsureDirectoryExists(android::base::Dirname(base_pkg))); + struct stat st; + CF_EXPECTF(stat(src_path.c_str(), &st) == 0, "Failed to stat('{}'): {}", + src_path, StrError(errno)); + // bin/cvd is sensitive to location, uses readlink("/proc/self/exe"). + // Otherwise, the input may be either a file in the source tree or a bazel + // artifact. If we hard link the file in the source tree, bazel will try to + // chmod it ( https://github.com/bazelbuild/bazel/issues/5588 ). In the case + // of a hard link it will propagate to the source directory, which can be + // surprising. As a heuristic, this tries to distinguish source files from + // generated artifacts by seeing if the permissions are already what bazel + // assigns to generated artifacts. If the source file looks like a bazel + // artifact, it is copied instead of hard linked. + if (pkg_path == "bin/cvd" || (st.st_mode & 0777) != 0555) { + CF_EXPECT(CopyWithAttributes(src_path, base_pkg)); + } else { + CF_EXPECT(LinkOrCopy(src_path, base_pkg)); + } + } + + for (const auto& [sym, src] : args.PackageFileSymlinkToPackageFile()) { + const std::string base_sym = absl::StrCat(args.BaseDir(), "/", sym); + CF_EXPECT(EnsureDirectoryExists(android::base::Dirname(base_sym))); + CF_EXPECT(Symlink(GetRelativePathForLink(src, sym), base_sym)); + } + + return {}; +} + +} // namespace +} // namespace cuttlefish + +int main(int argc, char** argv) { + std::vector args(argv + 1, argv + argc); + cuttlefish::Result res = cuttlefish::PackagerMain(std::move(args)); + if (!res.has_value()) { + std::cerr << res.error(); + return 1; + } + return 0; +} diff --git a/base/cvd/cuttlefish/package/rules.bzl b/base/cvd/cuttlefish/package/rules.bzl index 0f305a2691b..109f15ed2b2 100644 --- a/base/cvd/cuttlefish/package/rules.bzl +++ b/base/cvd/cuttlefish/package/rules.bzl @@ -4,114 +4,86 @@ def _file_from_label(l): fail(msg = "Unexpected number of files in target {}: {}".format(l, len(files))) return files[0] -def _get_common_prefix(paths): - if not paths: - return "" - prefix = paths[0].split("/") - for path in paths[1:]: - path_components = path.split("/") - common_length = min(len(prefix), len(path_components)) - prefix = [prefix[i] for i in range(common_length) if prefix[i] == path_components[i]] - if not prefix: - break - - return "/".join(prefix) - -def _remove_prefix(path, path_prefix): - # Empty prefix is a no-op (caller's common-prefix computation may return - # "" when two paths share no leading directory, e.g. lib64/x vs bin/y). - if not path_prefix: - return path - - path_parts = path.split("/") - - for path_prefix_part in path_prefix.split("/"): - if path_parts[0] != path_prefix_part: - fail("{} is not prefixed by {}".format(path, path_prefix)) - path_parts.pop(0) - - return "/".join(path_parts) - -# Finds the relative path needed to create a symlink between the link file at -# `link_path` to the link target at `link_target_path`. -# -# Example: -# _get_relative_path_for_link( -# link_target_path = "bin/graphics_detector", -# link_path = "bin/aarch64-linux-gnu/gfxstream_graphics_detector", -# ) == "../graphics_detector" -def _get_relative_path_for_link(link_target_path, link_path): - common_path = _get_common_prefix([link_target_path, link_path]) - - relative_link_path = _remove_prefix(link_path, common_path) - relative_link_path_depth = len(relative_link_path.split("/")) - 1 - - relative_link_target_path = _remove_prefix(link_target_path, common_path) - - return "".join(["../" for i in range(relative_link_path_depth)]) + relative_link_target_path +""" +Due to https://www.github.com/bazelbuild/bazel/issues/21782 , a +declare_directory target cannot overlap with any other declare_file targets. +That means the entire directory has to be created at once, pushing in the +direction of implementing a tool that creates the complete packaged directory. +Without using declare_directory and regenerating the whole directory, the +implementation risks stale files staying in the build directory, making builds +less hermetic. +""" def _package_files_impl(ctx): - default_outputs = list() - - path_to_declared_file = dict() - - for (dst, src) in ctx.attr.package_file_to_src.items(): - out_file = ctx.actions.declare_file(ctx.attr.base_dir + '/' + dst) + inputs = list() - path_to_declared_file[dst] = out_file + base_dir = ctx.actions.declare_directory(ctx.attr.base_dir) - if dst == ctx.attr.executable: - executable = out_file - default_outputs.append(out_file) + args = ["--base_dir=" + base_dir.path] + for (dst, src) in ctx.attr.package_file_to_src.items(): input_file = _file_from_label(src) - ctx.actions.run_shell( - mnemonic = "MakeOutputDir", - outputs = [out_file], - inputs = [input_file], - command = "mkdir -p " + out_file.dirname + " && cp " + input_file.path + " " + out_file.path, - ) + inputs.append(input_file) + args.append("--package_file_to_src=" + dst + "=" + input_file.path) for (dst, target) in ctx.attr.package_file_symlink_to_package_file.items(): - dst_file = ctx.actions.declare_file(ctx.attr.base_dir + '/' + dst) - - default_outputs.append(dst_file) - - src_file = path_to_declared_file.get(target) - if src_file == None: - fail(msg = "Package file \"{}\" in \"package_file_symlink_to_package_file\" does not exist as a key in \"package_file_to_src\"".format(target)) - - relative_target = _get_relative_path_for_link(target, dst) + args.append("--package_file_symlink_to_package_file=" + dst + "=" + target) - # https://github.com/bazelbuild/bazel/issues/14224: `ctx.actions.symlink()` is not used - # here because that appears to a link using the absolute path whereas links relative to - # the package are desired here. - ctx.actions.run_shell( - mnemonic = "MakeOutputDir", - outputs = [dst_file], - inputs = [src_file], - command = "mkdir -p " + dst_file.dirname + " && ln -s " + relative_target + " " + dst_file.path, - ) + ctx.actions.run( + mnemonic = "MakeOutputDir", + outputs = [base_dir], + inputs = inputs, + executable = _file_from_label(ctx.attr._packager), + arguments = args, + ) return [ - DefaultInfo( - executable = executable, - files = depset(default_outputs), - ), + DefaultInfo(files = depset([base_dir])), ] package_files = rule( attrs = { "base_dir": attr.string(), - "executable": attr.string(), "package_file_to_src": attr.string_keyed_label_dict( doc = "File paths within the generated package archive to the target providing the file.", allow_files = True), "package_file_symlink_to_package_file": attr.string_dict( doc = """File path within the generated package archive for a symlink pointing to another package file created by "package_file_to_src".""", ), + "_packager": attr.label( + executable = True, + cfg = "exec", + default = Label("//cuttlefish/package:packager"), + ), }, - executable = True, implementation = _package_files_impl, ) +def _package_executable_impl(ctx): + executable_link = ctx.actions.declare_symlink(ctx.attr.name) + + base_dir = _file_from_label(ctx.attr.package[DefaultInfo]) + if base_dir.dirname != executable_link.dirname: + fail("package_files and package_executable must be in the same directory") + + ctx.actions.run_shell( + mnemonic = "OutputSymlink", + outputs = [executable_link], + inputs = ctx.attr.package[DefaultInfo].files, + command = "ln -s " + base_dir.basename + "/" + ctx.attr.executable + " " + executable_link.path, + ) + return [ + DefaultInfo( + executable = executable_link, + files = depset([executable_link]) + ), + ] + +package_executable = rule( + attrs = { + "executable": attr.string(), + "package": attr.label(), + }, + executable = True, + implementation = _package_executable_impl, +) diff --git a/base/debian/rules b/base/debian/rules index 73c3554dc16..b89add0a174 100755 --- a/base/debian/rules +++ b/base/debian/rules @@ -82,10 +82,20 @@ override_dh_installsystemd: dh_installsystemd --name=cuttlefish-host-resources dh_installsystemd +.PHONY: cf_bazel_build +cf_bazel_build: + # This needs to be a separate step from override_dh_auto_build to delay the + # shell expansion until after bazel runs + cd cvd && bazel build ${remote_cache_arg} ${disk_cache_arg} ${compilation_mode} ${conlyopts} ${copts} ${cxxopts} ${linkopts} -c opt 'cuttlefish/package:common' 'cuttlefish/package:defaults' 'cuttlefish/package:metrics' --spawn_strategy=local --workspace_status_command=../stamp_helper.sh --build_tag_filters=-clang-tidy + # the `--workspace_status_command` flag path depends on the current working directory of base/cvd .PHONY: override_dh_auto_build -override_dh_auto_build: - cd cvd && bazel build ${remote_cache_arg} ${disk_cache_arg} ${compilation_mode} ${conlyopts} ${copts} ${cxxopts} ${linkopts} -c opt 'cuttlefish/package:cvd' 'cuttlefish/package:defaults' 'cuttlefish/package:metrics' --spawn_strategy=local --workspace_status_command=../stamp_helper.sh --build_tag_filters=-clang-tidy +override_dh_auto_build: cf_bazel_build + # https://github.com/bazelbuild/bazel/issues/5588 + # Bazel will chmod all the outputs automatically. This has a bad interaction + # with cuttlefish-base.install which tries to insert additional files into + # the output directories. + chmod +w $(shell find cvd/bazel-out/k8-opt/bin/cuttlefish/package/cuttlefish-common -type d) $(shell find cvd/bazel-out/aarch64-opt/bin/cuttlefish/package/cuttlefish-common -type d) $(shell find cvd/bazel-out/riscv64-opt/bin/cuttlefish/package/cuttlefish-common -type d) dh_auto_build # Only generate optimized DWARF if debug is enabled @@ -126,3 +136,9 @@ override_dh_fixperms: override_dh_installudev: dh_installudev --package=cuttlefish-base --priority=88 dh_installudev --remaining-packages + +# Deal with unwriteable files left in the output directory +.PHONY: override_dh_clean +override_dh_clean: + -chmod -R +w ${cuttlefish_common} + dh_clean