From f441f5ad8a8b6a70686c659728e82481383a879d Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Wed, 17 Jun 2026 23:50:07 +0200 Subject: [PATCH 01/12] Add experimental FastDeploy strategies Introduce FastDeploy2 and FastDeploy3 as opt-in fast deployment strategies for comparing adb push --sync and adb sync based file transfer paths. Add diagnostic timing counters to the legacy FastDeploy path for preliminary benchmarking. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tasks/FastDeploy.cs | 52 + .../Tasks/FastDeploy2.cs | 1267 +++++++++++++++++ .../Tasks/FastDeploy3.cs | 259 ++++ .../Xamarin.Android.Common.Debugging.targets | 56 + 4 files changed, 1634 insertions(+) create mode 100644 src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs create mode 100644 src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy3.cs diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy.cs index aa940b1d75d..a099fc06f19 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy.cs @@ -111,6 +111,16 @@ private class DiagnosticData { { "deploy.supports.fastdev", "True" }, { "deploy.systemapp", "False" }, { "deploy.duration.ms", "0" }, + { "deploy.fastdeploy.tools.install.ms", "" }, + { "deploy.fastdeploy.remote.mkdir.ms", "" }, + { "deploy.fastdeploy.remote.find.ms", "" }, + { "deploy.fastdeploy.compare.ms", "" }, + { "deploy.fastdeploy.copy.ms", "" }, + { "deploy.fastdeploy.environment.copy.ms", "" }, + { "deploy.fastdeploy.stale.remove.ms", "" }, + { "deploy.fastdeploy.changed.files", "" }, + { "deploy.fastdeploy.skipped.files", "" }, + { "deploy.fastdeploy.stale.files", "" }, { "pii.deploy.error", "" }, { "pii.deploy.file", "" }, }; @@ -325,9 +335,12 @@ public async Task RunInstall () if (EmbedAssembliesIntoApk) return; + var phase = Stopwatch.StartNew (); if (!await InstallFastDevTools (ToolsFullPath)) { + SetDiagnosticElapsed ("deploy.fastdeploy.tools.install.ms", phase); return; } + SetDiagnosticElapsed ("deploy.fastdeploy.tools.install.ms", phase); if (FastDevFiles?.Any () ?? false) { await TerminateApp (); @@ -665,6 +678,7 @@ protected async Task DeployFastDevFiles (string toolPath, string overridePath) LZ4Level lz4level = LZ4Level.L03_HC; LogDiagnostic ("Calculating subdirectories"); + var phase = Stopwatch.StartNew (); HashSet directories = new HashSet (); directories.Add (overridePath); foreach (var file in FastDevFiles) { @@ -694,13 +708,19 @@ protected async Task DeployFastDevFiles (string toolPath, string overridePath) args.Add (dir); } await Device.RunAs (packageInfo, args); + SetDiagnosticElapsed ("deploy.fastdeploy.remote.mkdir.ms", phase); + phase.Restart (); string filelist = await Device.RunAs (packageInfo, $"{toolPath}/{FastDevFindTool}", DiagnosticLogging ? "-vd" : "-v", overridePath); + SetDiagnosticElapsed ("deploy.fastdeploy.remote.find.ms", phase); LogDiagnostic ($"{FastDevFindTool}: {filelist}"); string [] files = Array.Empty (); if (!(filelist.IndexOf ("error:", StringComparison.OrdinalIgnoreCase) >= 0)) { files = filelist.Split (new char [] { '\n' }, StringSplitOptions.RemoveEmptyEntries); } + int changedFiles = 0; + int skippedFiles = 0; + var compareWatch = Stopwatch.StartNew (); Dictionary fileData = new Dictionary (); foreach (var file in files) { // file size mtime if (file.IndexOf ("\t") == -1) { @@ -730,6 +750,7 @@ protected async Task DeployFastDevFiles (string toolPath, string overridePath) foreach (var file in FastDevFiles) { if (!File.Exists (file.ItemSpec)) { LogDebugMessage ($"File '{file.ItemSpec}' does not exists. Skipping."); + skippedFiles++; continue; } StartTiming (); @@ -737,6 +758,7 @@ protected async Task DeployFastDevFiles (string toolPath, string overridePath) string abi = AndroidRidAbiHelper.GetNativeLibraryAbi (file); if (abi != PrimaryCpuAbi) { LogDebugMessageWithTiming ($"NotifySync SkipCopyFile {file.ItemSpec} abi not suitable for this device."); + skippedFiles++; continue; } } @@ -758,12 +780,18 @@ protected async Task DeployFastDevFiles (string toolPath, string overridePath) if (!modified) { LogDebugMessageWithTiming ($"NotifySync SkipCopyFile {file.ItemSpec}=>{targetPath} file is up to date."); fileData.Remove (targetPath); + skippedFiles++; continue; } + changedFiles++; + compareWatch.Stop (); + phase.Restart (); if (!await DeployFileWithFastDevTool (file, toolPath, overridePath, lz4level, modifiedDateTime)) { diagnosticData.SetProperty ("deploy.result", "Failed"); return; } + AddDiagnosticElapsed ("deploy.fastdeploy.copy.ms", phase); + compareWatch.Start (); LogDebugMessageWithTiming ($"NotifySync CopyFile {file.ItemSpec}."); LogDiagnostic ($"Local Modified Time '{modifiedDateTime.ToUnixTimeMilliseconds ()}' is newer than '{remoteDateTime.ToUnixTimeMilliseconds ()}'."); fileData.Remove (targetPath); @@ -774,15 +802,26 @@ protected async Task DeployFastDevFiles (string toolPath, string overridePath) if (fileData.ContainsKey (targetPath)) { remoteDateTime = fileData [targetPath].mtime; } + compareWatch.Stop (); + phase.Restart (); await DeployEnvironmentFiles (EnvironmentFiles, toolPath, overridePath, targetPath, remoteDateTime); + AddDiagnosticElapsed ("deploy.fastdeploy.environment.copy.ms", phase); + compareWatch.Start (); fileData.Remove (targetPath); } + compareWatch.Stop (); + SetDiagnosticElapsed ("deploy.fastdeploy.compare.ms", compareWatch); + diagnosticData.SetProperty ("deploy.fastdeploy.changed.files", changedFiles); + diagnosticData.SetProperty ("deploy.fastdeploy.skipped.files", skippedFiles); + diagnosticData.SetProperty ("deploy.fastdeploy.stale.files", fileData.Count); + phase.Restart (); foreach (var file in fileData.Keys) { // we need to remove unknown files from the .__override__ path string targetFile = $"{file.Replace ("./", "")}"; LogDebugMessage ($"Remove redundant file {OverrideFullPath}/{targetFile}"); await Device.RunAs (packageInfo, "rm", "-Rf", $"{OverrideFullPath}/{targetFile}"); } + SetDiagnosticElapsed ("deploy.fastdeploy.stale.remove.ms", phase); // clean up the temp folder if we are not using the xamarin.sync tool if (!packageInfo.SupportsFastDev) await Device.RunShellCommand ("rm", "-Rf", XAToolsTempPath); @@ -989,6 +1028,19 @@ bool RaiseRunAsError (string error) string GetFullPath (string dir) => Path.IsPathRooted (dir) ? dir : Path.GetFullPath (Path.Combine (WorkingDirectory, dir)); + void SetDiagnosticElapsed (string key, Stopwatch stopwatch) + { + diagnosticData.SetProperty (key, stopwatch.ElapsedMilliseconds); + } + + void AddDiagnosticElapsed (string key, Stopwatch stopwatch) + { + if (!long.TryParse (diagnosticData.Properties [key], out long current)) { + current = 0; + } + diagnosticData.SetProperty (key, current + stopwatch.ElapsedMilliseconds); + } + static string GetErrorCode (string message) { foreach (var errorCode in error_codes) diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs new file mode 100644 index 00000000000..f4ab215bddd --- /dev/null +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs @@ -0,0 +1,1267 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; +using System.Threading; +using System.Threading.Tasks; + +using Microsoft.Android.Build.Tasks; +using Microsoft.Build.Framework; +using Mono.AndroidTools; +using Mono.AndroidTools.Util; +using Xamarin.Android.Build.Debugging.Tasks.Properties; + +namespace Xamarin.Android.Tasks +{ + public class FastDeploy2 : AsyncTask + { + const string OverridePath = "files/.__override__"; + const int StaleFileRemovalBatchSize = 100; + const int CopyBatchSize = 25; + + public override string TaskPrefix => "FD2"; + + public string AdbTarget { get; set; } + public string UploadFlagFile { get; set; } + public bool EmbedAssembliesIntoApk { get; set; } + public bool ReInstall { get; set; } = false; + + [Required] + public string PackageName { get; set; } + + public string PackageFile { get; set; } + + public string PrimaryCpuAbi { get; set; } + public string ToolsAbi { get; set; } + + public ITaskItem [] FastDevFiles { get; set; } + + public bool PreserveUserData { get; set; } = true; + + [Required] + public string FastDevToolPath { get; set; } + + [Required] + public string ToolVersion { get; set; } + + public bool DiagnosticLogging { get; set; } = false; + + public bool UsingAndroidNETSdk { get; set; } + + public string UserID { get; set; } + + public bool IsTestOnly { get; set; } + + [Required] + public string IntermediateOutputPath { get; set; } + + public ITaskItem [] EnvironmentFiles { get; set; } + + public string AdbToolPath { get; set; } + + public string AdbToolExe { get; set; } + + public string AdbPushCompressionAlgorithm { get; set; } = "any"; + + AndroidDevice Device; + PackageInfo packageInfo = new PackageInfo (); + DateTime lastUpload = DateTime.MinValue; + Queue diagnosticLogs = new Queue (); + DiagnosticData diagnosticData = new DiagnosticData (); + + protected virtual string RemoteStagingRoot => "/tmp/fastdev2"; + + string OverrideFullPath { + get { return packageInfo.IsSystemApplication ? $"{packageInfo.InternalPath}/{OverridePath}" : OverridePath; } + } + + class PackageInfo { + string internalPath = null; + public string InternalPath { + get { return internalPath; } + set { internalPath = value?.Trim () ?? null; } + } + + public bool IsSystemApplication { get; set; } = false; + public bool AdbIsRoot { get; set; } = false; + public string UserId { get; set; } = null; + public string PackageName { get; set; } = null; + } + + class DiagnosticData { + [JsonPropertyName ("Task")] + public string Task { get; set; } = nameof (FastDeploy2); + + [JsonPropertyName ("Properties")] + public Dictionary Properties { get; set; } = new Dictionary () { + { "target.prop.ro.product.build.version.sdk", "" }, + { "target.prop.ro.product.cpu.abilist", "" }, + { "target.prop.ro.product.manufacturer", "" }, + { "target.prop.ro.product.model", "" }, + { "target.prop.ro.product.cpu.abi", "" }, + { "deploy.error.code", "" }, + { "deploy.tool", "adb push" }, + { "deploy.result", "Success" }, + { "deploy.supports.fastdev", "True" }, + { "deploy.systemapp", "False" }, + { "deploy.duration.ms", "0" }, + { "deploy.fastdeploy2.adb.pushed.files", "" }, + { "deploy.fastdeploy2.adb.skipped.files", "" }, + { "deploy.fastdeploy2.changed.files", "" }, + { "deploy.fastdeploy2.stale.files", "" }, + { "deploy.fastdeploy2.local.stage.ms", "" }, + { "deploy.fastdeploy2.remote.mkdir.ms", "" }, + { "deploy.fastdeploy2.remote.staging.cleanup.ms", "" }, + { "deploy.fastdeploy2.upload.ms", "" }, + { "deploy.fastdeploy2.staging.stat.ms", "" }, + { "deploy.fastdeploy2.override.stat.ms", "" }, + { "deploy.fastdeploy2.compare.ms", "" }, + { "deploy.fastdeploy2.stale.remove.ms", "" }, + { "deploy.fastdeploy2.override.mkdir.ms", "" }, + { "deploy.fastdeploy2.override.copy.ms", "" }, + { "deploy.fastdeploy3.sync.list.ms", "" }, + { "deploy.fastdeploy3.sync.list.files", "" }, + { "deploy.fastdeploy3.override.list.ms", "" }, + { "deploy.fastdeploy3.missing.files", "" }, + { "pii.deploy.error", "" }, + { "pii.deploy.file", "" }, + }; + + internal void SetProperty (string key, bool? value) + { + Properties [key] = value?.ToString () ?? "False"; + } + + internal void SetProperty (string key, int? value) + { + Properties [key] = value?.ToString () ?? "-1"; + } + + internal void SetProperty (string key, long? value) + { + Properties [key] = value?.ToString () ?? "-1"; + } + + internal void SetProperty (string key, string value) + { + Properties [key] = value ?? "unknown"; + } + } + + class RemoteFileInfo { + public long Size { get; set; } + public long ModifiedTime { get; set; } + } + + void DebugHandler (string task, string message) + { + LogDiagnostic ($"DEBUG {task} {message}"); + } + + public override bool Execute () + { + Device = AndroidHelper.ParseTarget (AdbTarget, LogMessage, LogCodedError, logErrors: true, engine4: BuildEngine4); + if (Device == null) { + PrintDiagnostics (); + return false; + } + LogMessage ($"Found device: {Device.ID}"); + + if (string.IsNullOrEmpty (PrimaryCpuAbi) && !EmbedAssembliesIntoApk) { + PrintDiagnostics (); + LogCodedError ("XA0010", Resources.XA0010_NoAbi, Device.ID); + return false; + } + + var flagFilePath = GetFullPath (UploadFlagFile); + lastUpload = File.GetLastWriteTimeUtc (flagFilePath); + LogDiagnostic ($"LastWriteTime of `{flagFilePath}`: {lastUpload}"); + diagnosticData.Task = GetType ().Name; + + var lifetime = RegisteredTaskObjectLifetime.AppDomain; + var key = ProjectSpecificTaskObjectKey ($"{Device.ID}_{PackageName}_{GetType ().Name}"); + if (!File.Exists (UploadFlagFile)) { + packageInfo = new PackageInfo (); + } else { + packageInfo = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (key, lifetime) ?? new PackageInfo (); + } + + AndroidLogger.Debug += DebugHandler; + try { + return base.Execute (); + } finally { + BuildEngine4.RegisterTaskObjectAssemblyLocal (key, packageInfo, lifetime, allowEarlyCollection: false); + AndroidLogger.Debug -= DebugHandler; + } + } + + public async override Task RunTaskAsync () + { + var sw = Stopwatch.StartNew (); + try { + await RunInstall (); + } catch { + PrintDiagnostics (); + throw; + } finally { + sw.Stop (); + SaveDiagnosticData (sw.ElapsedMilliseconds); + } + } + + async Task RunInstall () + { + await Device.EnsureProperties (CancellationToken).ConfigureAwait (false); + + diagnosticData.SetProperty ("target.prop.ro.product.build.version.sdk", Device.Properties?.BuildVersionSdk); + diagnosticData.SetProperty ("target.prop.ro.product.cpu.abilist", string.Join (";", Device.Properties?.ProductCpuAbiList ?? Array.Empty ())); + diagnosticData.SetProperty ("target.prop.ro.product.cpu.abi", PrimaryCpuAbi); + diagnosticData.SetProperty ("target.prop.ro.product.manufacturer", Device.Properties?.ProductManufacturer); + diagnosticData.SetProperty ("target.prop.ro.product.model", Device.Properties?.ProductModel); + + string redirectStdio = Device.Properties.Get ("log.redirect-stdio"); + if (redirectStdio != null && string.Equals ("true", redirectStdio.Trim (), StringComparison.OrdinalIgnoreCase)) { + LogFastDeploy2Error ("XA0128", Resources.XA0128_RedirectStdioIsEnabled); + return; + } + + string runAsDisabled = Device.Properties.Get ("ro.boot.disable_runas"); + if (runAsDisabled != null && string.Equals ("true", runAsDisabled.Trim (), StringComparison.OrdinalIgnoreCase)) { + LogFastDeploy2Error ("XA0131", Resources.XA0131_DeveloperModeNotEnabled); + return; + } + + await CheckAppInstalledAndDebuggable (PackageName); + + if (EmbedAssembliesIntoApk) { + await RemoveOverrideDirectory (); + } + + if (ReInstall && !string.IsNullOrEmpty (PackageFile)) { + await Device.UninstallPackage (PackageName, PreserveUserData, CancellationToken); + } + + if (!string.IsNullOrEmpty (PackageFile) && + (packageInfo.InternalPath.IndexOf ("unknown", StringComparison.OrdinalIgnoreCase) >= 0 || ReInstall || IsPackageFileOutOfDate ())) { + try { + await InstallPackage (); + } catch (Exception ex) { + LogFastDeploy2Error (GetErrorCode (ex), ex.ToString ()); + return; + } + if (!EmbedAssembliesIntoApk && packageInfo.InternalPath.IndexOf ("unknown", StringComparison.OrdinalIgnoreCase) >= 0) { + packageInfo.InternalPath = null; + await CheckAppInstalledAndDebuggable (PackageName); + if (RaiseRunAsError (packageInfo.InternalPath)) { + return; + } + } + } + + if (EmbedAssembliesIntoApk) + return; + + if ((FastDevFiles?.Length ?? 0) == 0 && (EnvironmentFiles?.Length ?? 0) == 0) { + return; + } + + await TerminateApp (); + await DeployFastDevFilesWithAdbPush (OverrideFullPath); + } + + bool IsPackageFileOutOfDate () + { + var packageFile = GetFullPath (PackageFile); + var lastPackage = File.GetLastWriteTimeUtc (packageFile); + LogDiagnostic ($"LastWriteTime of `{packageFile}`: {lastPackage}"); + return lastUpload < lastPackage; + } + + async Task CheckAppInstalledAndDebuggable (string packageName) + { + packageInfo.UserId = UserID; + packageInfo.PackageName = packageName; + await EnsureUserIsRunning (); + packageInfo.InternalPath = packageInfo.InternalPath ?? await RunAs ("pwd"); + if (packageInfo.InternalPath.IndexOf ("Permission denied", StringComparison.OrdinalIgnoreCase) >= 0) { + packageInfo.InternalPath = await RunAs ("readlink", "-f", "."); + } + if (packageInfo.InternalPath.IndexOf ("not an application", StringComparison.OrdinalIgnoreCase) >= 0) { + LogDiagnostic ($"Package {packageInfo.PackageName} is a system application."); + packageInfo.IsSystemApplication = true; + diagnosticData.SetProperty ("deploy.systemapp", value: true); + string whoami = await Device.RunShellCommand ("whoami"); + packageInfo.AdbIsRoot = whoami.Trim () == "root"; + LogDiagnostic ($"using {(packageInfo.AdbIsRoot ? "root" : $"su {packageInfo.UserId}")} to install fast deployment files."); + packageInfo.InternalPath = $"/data/user/{(packageInfo.UserId ?? "0")}/{packageInfo.PackageName}"; + return; + } + if (packageInfo.InternalPath.IndexOf ("not debuggable", StringComparison.OrdinalIgnoreCase) >= 0) { + LogDiagnostic ($"Package {packageInfo.PackageName} was not debuggable. Forcing ReInstall"); + ReInstall = true; + return; + } + if (packageInfo.InternalPath.IndexOf ("unknown", StringComparison.OrdinalIgnoreCase) >= 0) { + LogDiagnostic ($"Package {packageInfo.PackageName} was not installed."); + return; + } + if (packageInfo.InternalPath.IndexOf ("Permission denied", StringComparison.OrdinalIgnoreCase) >= 0) { + LogDiagnostic ("run-as not supported on this device."); + diagnosticData.SetProperty ("deploy.supports.fastdev", value: false); + } + } + + async Task EnsureUserIsRunning () + { + var userId = (UserID ?? "").Trim (); + if (userId.Length == 0 || (int.TryParse (userId, out var id) && id == 0)) { + return; + } + LogDiagnostic ($"Ensuring Android user {userId} is in the 'running' state before run-as queries."); + string output = await Device.RunShellCommand (CancellationToken, "am", "start-user", "-w", userId); + LogDiagnostic ($"'am start-user -w {userId}' returned: {(string.IsNullOrWhiteSpace (output) ? "" : output.Trim ())}"); + } + + async Task InstallPackage () + { + LogDebugMessage ($"Installing Package {PackageName}"); + try { + await Device.PushAndInstallPackageAsync (new PushAndInstallCommand { + ApkFile = PackageFile, + PackageName = PackageName, + ReInstall = ReInstall, + User = UserID, + TestOnly = IsTestOnly, + }, token: CancellationToken); + LogDebugMessage ($"Installed Package {PackageName}."); + } catch (Exception exception) { + var ex = exception; + if (exception is AggregateException aex) { + ex = aex.Flatten ().InnerException; + } + if (!await ShouldThrowIfPackageInstallFailed (ex as PackageAlreadyExistsException)) { + LogDebugMessage ($"Installed Package {PackageName}."); + return; + } + throw; + } + } + + async Task ShouldThrowIfPackageInstallFailed (PackageAlreadyExistsException e) + { + if (e == null) + return true; + + int s = (e.PackageFile ?? "").LastIndexOf ('/'); + string apkBasename = s >= 0 ? e.PackageFile.Substring (s + 1) : e.PackageFile; + + if (apkBasename != Path.GetFileName (PackageFile)) + return false; + + LogDebugMessage (string.Format ("Package '{0}' already exists. Retrying...", PackageName)); + try { + await Device.DeleteFile (e.PackageFile, true, CancellationToken); + } catch { + } + bool preserveData = !(e is RequiresUninstallException); + LogDebugMessage (string.Format ("Forcing complete uninstall of '{0}'... Preserving Data: {1}", PackageName, preserveData)); + var uninstallCommand = new PmUninstallCommand () { PackageName = PackageName, User = UserID, PreserveData = preserveData }; + await Device.UninstallPackage (uninstallCommand, cancellationToken: CancellationToken); + LogDebugMessage (string.Format ("Installing '{0}'...", PackageName)); + await Device.PushAndInstallPackageAsync (new PushAndInstallCommand { + ApkFile = PackageFile, + PackageName = PackageName, + ReInstall = false, + User = UserID + }, token: CancellationToken); + return false; + } + + async Task RemoveOverrideDirectory () + { + await RunAs ("rm", "-Rf", OverrideFullPath); + } + + async Task TerminateApp () + { + var pid = await Device.GetProcessId (PackageName, CancellationToken); + if (pid == 0) { + LogDebugMessage ($"{PackageName} was not running, skipping kill"); + return; + } + LogDebugMessage ($"Terminating {PackageName}..."); + await Device.KillProcessAndWaitForExit (PackageName, CancellationToken); + LogDebugMessage ($"{PackageName} Terminated."); + } + + protected virtual async Task DeployFastDevFilesWithAdbPush (string overridePath) + { + string stagingDirectory = GetLocalStagingDirectory (); + var phase = Stopwatch.StartNew (); + var stagedFiles = PrepareAdbPushStagingDirectory (stagingDirectory); + SetDiagnosticElapsed ("deploy.fastdeploy2.local.stage.ms", phase); + if (stagedFiles.Count == 0) { + LogDiagnostic ("No FastDev files were staged for adb push deployment."); + return true; + } + + string remoteStagingPath = GetRemoteAdbPushStagingPath (); + phase.Restart (); + string output = await Device.RunShellCommand (CancellationToken, "mkdir", "-p", remoteStagingPath); + SetDiagnosticElapsed ("deploy.fastdeploy2.remote.mkdir.ms", phase); + if (IsShellError (output, "mkdir")) { + LogFastDeploy2Error ("XA0129", output, remoteStagingPath); + return false; + } + + if (!await RemoveStaleRemoteStagingFiles (remoteStagingPath, stagedFiles)) { + return false; + } + + phase.Restart (); + if (!await UploadStagingDirectory (stagingDirectory, remoteStagingPath)) { + return false; + } + SetDiagnosticElapsed ("deploy.fastdeploy2.upload.ms", phase); + + phase.Restart (); + var stagedFileData = await GetRemoteFileData (remoteStagingPath, runAs: false); + SetDiagnosticElapsed ("deploy.fastdeploy2.staging.stat.ms", phase); + if (stagedFileData == null) { + return false; + } + + phase.Restart (); + var overrideFileData = await GetRemoteFileData (overridePath, runAs: true); + SetDiagnosticElapsed ("deploy.fastdeploy2.override.stat.ms", phase); + if (overrideFileData == null) { + return false; + } + + if (!await RemoveStaleOverrideFiles (overridePath, stagedFileData, overrideFileData)) { + return false; + } + + return await CopyChangedFiles (remoteStagingPath, overridePath, stagedFileData, overrideFileData); + } + + protected HashSet PrepareAdbPushStagingDirectory (string stagingDirectory) + { + if (Directory.Exists (stagingDirectory)) { + Directory.Delete (stagingDirectory, recursive: true); + } + Directory.CreateDirectory (stagingDirectory); + + var stagedFiles = new HashSet (StringComparer.Ordinal); + foreach (var file in FastDevFiles ?? Array.Empty ()) { + if (!File.Exists (file.ItemSpec)) { + LogDebugMessage ($"File '{file.ItemSpec}' does not exists. Skipping."); + continue; + } + if (Path.GetExtension (file.ItemSpec) == ".so") { + string abi = AndroidRidAbiHelper.GetNativeLibraryAbi (file); + if (abi != PrimaryCpuAbi) { + LogDebugMessage ($"NotifySync SkipCopyFile {file.ItemSpec} abi not suitable for this device."); + continue; + } + } + + string targetPath = GetAdbPushTargetPath (file); + string destination = GetStagingFilePath (stagingDirectory, targetPath); + Directory.CreateDirectory (Path.GetDirectoryName (destination)); + File.Copy (file.ItemSpec, destination, overwrite: true); + File.SetLastWriteTimeUtc (destination, File.GetLastWriteTimeUtc (file.ItemSpec)); + stagedFiles.Add (targetPath.Replace ("\\", "/")); + LogDiagnostic ($"Staged {file.ItemSpec} => {targetPath}"); + } + + if (EnvironmentFiles?.Length > 0) { + string targetPath = $"{PrimaryCpuAbi}/environment"; + string destination = GetStagingFilePath (stagingDirectory, targetPath); + Directory.CreateDirectory (Path.GetDirectoryName (destination)); + byte [] environmentData = CreateEnvironmentFileData (EnvironmentFiles, out DateTime newestFileDateTime); + if (environmentData.Length > 0) { + File.WriteAllBytes (destination, environmentData); + File.SetLastWriteTimeUtc (destination, newestFileDateTime); + stagedFiles.Add (targetPath); + LogDiagnostic ($"Staged @(AndroidEnvironment) files => {targetPath}"); + } + } + + return stagedFiles; + } + + string GetAdbPushTargetPath (ITaskItem file) + { + string targetPath = file.GetMetadata ("TargetPath"); + if (string.IsNullOrEmpty (targetPath)) { + LogDiagnostic ($"'TargetPath' meta data not found on '{file.ItemSpec}'. Falling back to'DestinationSubPath'"); + targetPath = file.GetMetadata ("DestinationSubPath"); + } + if (!string.IsNullOrEmpty (targetPath)) { + return targetPath.Replace ("\\", "/"); + } + return Path.GetFileName (file.ItemSpec); + } + + static string GetStagingFilePath (string stagingDirectory, string targetPath) + { + string fullStagingDirectory = Path.GetFullPath (stagingDirectory); + string destination = Path.GetFullPath (Path.Combine (fullStagingDirectory, targetPath.Replace ('/', Path.DirectorySeparatorChar))); + string stagingPrefix = fullStagingDirectory.EndsWith (Path.DirectorySeparatorChar.ToString (), StringComparison.Ordinal) ? + fullStagingDirectory : + fullStagingDirectory + Path.DirectorySeparatorChar; + if (!destination.StartsWith (stagingPrefix, StringComparison.Ordinal)) { + throw new InvalidOperationException ($"FastDev target path '{targetPath}' escapes staging directory '{stagingDirectory}'."); + } + return destination; + } + + byte [] CreateEnvironmentFileData (ITaskItem [] environments, out DateTime newestFileDateTime) + { + int maxKeyLength = 0; + int maxValueLength = 0; + newestFileDateTime = DateTime.MinValue; + var data = new Dictionary (); + foreach (ITaskItem env in environments ?? Array.Empty ()) { + if (!File.Exists (env.ItemSpec)) + continue; + DateTime modifiedDateTime = File.GetLastWriteTimeUtc (env.ItemSpec); + if (modifiedDateTime > newestFileDateTime) + newestFileDateTime = modifiedDateTime; + foreach (string line in File.ReadLines (env.ItemSpec)) { + if (string.IsNullOrEmpty (line)) + continue; + int index = line.IndexOf ('='); + if (index == -1) { + LogDebugMessage ($"Skipping invalid environment line: {line}"); + continue; + } + var key = line.Substring (0, index); + var value = line.Substring (index + 1); + maxKeyLength = Math.Max (maxKeyLength, key.Length); + maxValueLength = Math.Max (maxValueLength, value.Length); + data [key] = value; + } + } + + if (newestFileDateTime == DateTime.MinValue) { + return Array.Empty (); + } + + maxKeyLength++; + maxValueLength++; + + using (var stream = new MemoryStream ()) + using (var binaryWriter = new BinaryWriter (stream, Encoding.ASCII)) { + binaryWriter.Write (Encoding.ASCII.GetBytes ("0x" + maxKeyLength.ToString ("X8") + '\0')); + binaryWriter.Write (Encoding.ASCII.GetBytes ("0x" + maxValueLength.ToString ("X8") + '\0')); + foreach (var kvp in data) { + binaryWriter.Write (Encoding.ASCII.GetBytes (kvp.Key.PadRight (maxKeyLength, '\0'))); + binaryWriter.Write (Encoding.ASCII.GetBytes (kvp.Value.PadRight (maxValueLength, '\0'))); + } + binaryWriter.Flush (); + return stream.ToArray (); + } + } + + protected async Task RemoveStaleRemoteStagingFiles (string remoteStagingPath, HashSet stagedFiles) + { + var phase = Stopwatch.StartNew (); + string filelist = await Device.RunShellCommand (CancellationToken, "find", remoteStagingPath, "-type", "f"); + if (IsShellError (filelist, "find")) { + LogFastDeploy2Error ("XA0129", filelist, remoteStagingPath); + return false; + } + + string prefix = remoteStagingPath.TrimEnd ('/') + "/"; + var staleFiles = new List (); + foreach (string line in filelist.Split (new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) { + string remoteFile = line.Trim (); + if (!remoteFile.StartsWith (prefix, StringComparison.Ordinal)) { + continue; + } + string relativePath = remoteFile.Substring (prefix.Length); + if (!stagedFiles.Contains (relativePath)) { + staleFiles.Add (remoteFile); + } + } + + for (int i = 0; i < staleFiles.Count; i += StaleFileRemovalBatchSize) { + var args = new List { "rm", "-f" }; + args.AddRange (staleFiles.Skip (i).Take (StaleFileRemovalBatchSize)); + string output = await Device.RunShellCommand (CancellationToken, args.ToArray ()); + if (IsShellError (output, "rm")) { + LogFastDeploy2Error ("XA0129", output, remoteStagingPath); + return false; + } + } + + SetDiagnosticElapsed ("deploy.fastdeploy2.remote.staging.cleanup.ms", phase); + return true; + } + + async Task> GetRemoteFileData (string rootPath, bool runAs) + { + string output; + if (runAs) { + output = await RunAs ("find", rootPath, "-type", "f", "-exec", "stat", "-c", "%n|%s|%Y", "{}", "+"); + if (RaiseRunAsError (output)) { + return null; + } + } else { + output = await Device.RunShellCommand (CancellationToken, "find", rootPath, "-type", "f", "-exec", "stat", "-c", "%n|%s|%Y", "{}", "+"); + } + + if (IsMissingDirectoryError (output)) { + return new Dictionary (StringComparer.Ordinal); + } + if (IsShellError (output, "find") || IsShellError (output, "stat")) { + LogFastDeploy2Error ("XA0129", output, rootPath); + return null; + } + + return ParseRemoteFileData (rootPath, output); + } + + Dictionary ParseRemoteFileData (string rootPath, string output) + { + var files = new Dictionary (StringComparer.Ordinal); + string prefix = rootPath.TrimEnd ('/') + "/"; + foreach (string line in output.Split (new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) { + var entries = line.Split (new char [] { '|' }, 3); + if (entries.Length != 3) { + LogDebugMessage ($"Ignoring remote file entry '{line}'. Line is incorrectly formatted."); + continue; + } + string remoteFile = entries [0].Trim (); + if (!remoteFile.StartsWith (prefix, StringComparison.Ordinal)) { + LogDebugMessage ($"Ignoring remote file entry '{line}'. Path is outside '{rootPath}'."); + continue; + } + if (!long.TryParse (entries [1].Trim (), out long size) || !long.TryParse (entries [2].Trim (), out long mtime)) { + LogDebugMessage ($"Ignoring remote file entry '{line}'. Size or timestamp is invalid."); + continue; + } + files [remoteFile.Substring (prefix.Length)] = new RemoteFileInfo { + Size = size, + ModifiedTime = mtime, + }; + } + return files; + } + + async Task RemoveStaleOverrideFiles (string overridePath, Dictionary stagedFiles, Dictionary overrideFiles) + { + var phase = Stopwatch.StartNew (); + var staleFiles = new List (); + foreach (var file in overrideFiles.Keys) { + if (!stagedFiles.ContainsKey (file)) { + staleFiles.Add ($"{overridePath}/{file}"); + } + } + + LogDiagnostic ($"FastDeploy2 removing {staleFiles.Count} stale override files."); + diagnosticData.SetProperty ("deploy.fastdeploy2.stale.files", staleFiles.Count); + for (int i = 0; i < staleFiles.Count; i += StaleFileRemovalBatchSize) { + var args = new List { "rm", "-f" }; + args.AddRange (staleFiles.Skip (i).Take (StaleFileRemovalBatchSize)); + string output = await RunAs (args.ToArray ()); + if (RaiseRunAsError (output) || IsShellError (output, "rm")) { + LogFastDeploy2Error ("XA0129", output, overridePath); + return false; + } + } + SetDiagnosticElapsed ("deploy.fastdeploy2.stale.remove.ms", phase); + return true; + } + + async Task CopyChangedFiles (string remoteStagingPath, string overridePath, Dictionary stagedFiles, Dictionary overrideFiles) + { + var phase = Stopwatch.StartNew (); + var changedFiles = new List (); + foreach (var file in stagedFiles) { + if (!overrideFiles.TryGetValue (file.Key, out RemoteFileInfo existing) || + existing.Size != file.Value.Size || + existing.ModifiedTime != file.Value.ModifiedTime) { + changedFiles.Add (file.Key); + } + } + SetDiagnosticElapsed ("deploy.fastdeploy2.compare.ms", phase); + + LogDiagnostic ($"FastDeploy2 copying {changedFiles.Count} changed override files."); + diagnosticData.SetProperty ("deploy.fastdeploy2.changed.files", changedFiles.Count); + var filesByDirectory = new Dictionary> (StringComparer.Ordinal); + foreach (string file in changedFiles) { + string directory = Path.GetDirectoryName (file)?.Replace ("\\", "/") ?? ""; + if (!filesByDirectory.TryGetValue (directory, out List files)) { + files = new List (); + filesByDirectory.Add (directory, files); + } + files.Add (file); + } + + foreach (var group in filesByDirectory) { + string targetDirectory = string.IsNullOrEmpty (group.Key) ? overridePath : $"{overridePath}/{group.Key}"; + phase.Restart (); + string output = await RunAs ("mkdir", "-p", targetDirectory); + AddDiagnosticElapsed ("deploy.fastdeploy2.override.mkdir.ms", phase); + if (RaiseRunAsError (output) || IsShellError (output, "mkdir")) { + LogFastDeploy2Error ("XA0129", output, targetDirectory); + return false; + } + + for (int i = 0; i < group.Value.Count; i += CopyBatchSize) { + var args = new List { "cp", "-p" }; + foreach (string file in group.Value.Skip (i).Take (CopyBatchSize)) { + args.Add ($"{remoteStagingPath}/{file}"); + } + args.Add (targetDirectory); + phase.Restart (); + output = await RunAs (args.ToArray ()); + AddDiagnosticElapsed ("deploy.fastdeploy2.override.copy.ms", phase); + if (RaiseRunAsError (output) || IsShellError (output, "cp")) { + LogFastDeploy2Error ("XA0129", output, targetDirectory); + return false; + } + } + } + + return true; + } + + protected void SetDiagnosticElapsed (string key, Stopwatch stopwatch) + { + diagnosticData.SetProperty (key, stopwatch.ElapsedMilliseconds); + } + + protected void AddDiagnosticElapsed (string key, Stopwatch stopwatch) + { + if (!long.TryParse (diagnosticData.Properties [key], out long current)) { + current = 0; + } + diagnosticData.SetProperty (key, current + stopwatch.ElapsedMilliseconds); + } + + protected void SetDiagnosticProperty (string key, int value) + { + diagnosticData.SetProperty (key, value); + } + + protected void SetDiagnosticProperty (string key, string value) + { + diagnosticData.SetProperty (key, value); + } + + protected virtual string GetLocalStagingDirectory () + { + return Path.Combine (GetFullPath (IntermediateOutputPath), "fastdeploy2"); + } + + protected virtual async Task UploadStagingDirectory (string stagingDirectory, string remoteStagingPath) + { + var args = new List { "push" }; + if (!string.IsNullOrEmpty (AdbPushCompressionAlgorithm)) { + args.Add ("-z"); + args.Add (AdbPushCompressionAlgorithm); + } + args.Add ("--sync"); + args.Add (Path.Combine (stagingDirectory, ".")); + args.Add (remoteStagingPath); + + var result = await RunAdbCommand (args.ToArray ()); + if (result.ExitCode != 0) { + LogFastDeploy2Error ("XA0129", result.Output, stagingDirectory); + return false; + } + SetAdbPushFileCounts (result.Output); + LogDiagnostic (result.Output); + return true; + } + + protected void SetAdbPushFileCounts (string output) + { + var match = AdbPushSummaryRegex.Match (output ?? ""); + if (!match.Success) { + return; + } + diagnosticData.SetProperty ("deploy.fastdeploy2.adb.pushed.files", match.Groups ["pushed"].Value); + diagnosticData.SetProperty ("deploy.fastdeploy2.adb.skipped.files", match.Groups ["skipped"].Value); + } + + protected async Task RunAdbCommand (params string [] arguments) + { + return await RunAdbCommand (arguments, environmentVariables: null); + } + + protected async Task RunAdbCommand (string [] arguments, Dictionary environmentVariables) + { + string adb = ResolveAdbPath (); + var processArguments = new ProcessArgumentBuilder (); + if (Device != null && !string.IsNullOrEmpty (Device.ID) && !string.Equals (Device.ID, "any", StringComparison.OrdinalIgnoreCase)) { + processArguments.AddQuoted ("-s"); + processArguments.AddQuoted (Device.ID); + } + processArguments.AddQuoted (arguments); + + var stdout = new StringBuilder (); + var stderr = new StringBuilder (); + var stdoutCompleted = new ManualResetEvent (false); + var stderrCompleted = new ManualResetEvent (false); + var psi = new ProcessStartInfo { + FileName = adb, + Arguments = processArguments.ToString (), + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + CreateNoWindow = true, + WindowStyle = ProcessWindowStyle.Hidden, + }; + if (environmentVariables != null) { + foreach (var kvp in environmentVariables) { + psi.EnvironmentVariables [kvp.Key] = kvp.Value; + } + } + + LogDiagnostic ($"Running adb: {psi.FileName} {psi.Arguments}"); + using (var process = new Process ()) { + process.StartInfo = psi; + process.OutputDataReceived += (sender, e) => { + if (e.Data != null) { + lock (stdout) { + stdout.AppendLine (e.Data); + } + LogDiagnostic (e.Data); + } else { + stdoutCompleted.Set (); + } + }; + process.ErrorDataReceived += (sender, e) => { + if (e.Data != null) { + lock (stderr) { + stderr.AppendLine (e.Data); + } + LogDiagnostic (e.Data); + } else { + stderrCompleted.Set (); + } + }; + + process.Start (); + process.BeginOutputReadLine (); + process.BeginErrorReadLine (); + using (CancellationToken.Register (() => { + try { + if (!process.HasExited) { + process.Kill (); + } + } catch (InvalidOperationException) { + } + })) { + await Task.Run (() => process.WaitForExit (), CancellationToken); + } + stdoutCompleted.WaitOne (TimeSpan.FromSeconds (30)); + stderrCompleted.WaitOne (TimeSpan.FromSeconds (30)); + return new AdbCommandResult { + ExitCode = process.ExitCode, + Output = $"{stdout}{stderr}".Trim (), + }; + } + } + + List BuildRunAsArgs () + { + List args = new List (); + if (packageInfo.IsSystemApplication) { + if (!packageInfo.AdbIsRoot) { + args.Add ("su"); + args.Add (packageInfo.UserId ?? "0"); + } + return args; + } + args.Add ("run-as"); + args.Add (packageInfo.PackageName); + if (!string.IsNullOrEmpty (packageInfo.UserId)) { + args.Add ("--user"); + args.Add (packageInfo.UserId); + } + return args; + } + + protected async Task RunAs (params string [] arguments) + { + List args = BuildRunAsArgs (); + args.AddRange (arguments); + string result = await Device.RunShellCommand (CancellationToken, args.ToArray ()); + LogDebugMessage ($"{arguments [0]} returned: {result}"); + return result; + } + + protected string ResolveAdbPath () + { + var exe = string.IsNullOrEmpty (AdbToolExe) ? "adb" : AdbToolExe; + return string.IsNullOrEmpty (AdbToolPath) ? exe : Path.Combine (AdbToolPath, exe); + } + + protected virtual string GetRemoteAdbPushStagingPath () + { + var user = string.IsNullOrEmpty (UserID) ? "0" : UserID; + return $"{RemoteStagingRoot}/{PackageName}/{user}"; + } + + protected void LogFastDeploy2Error (string errorCode, string error, string file = "") + { + LogDiagnosticDataError (errorCode, error, file); + PrintDiagnostics (); + if (errorCode == "XA0129") { + LogCodedError (errorCode, Resources.XA0129_ErrorDeployingFile, file); + } else { + LogCodedError (errorCode, error); + } + } + + protected void LogDiagnostic (string message) + { + if (DiagnosticLogging) { + LogDebugMessage (message); + return; + } + diagnosticLogs.Enqueue (message); + } + + void PrintDiagnostics () + { + while (diagnosticLogs.Count > 0) { + LogMessage (diagnosticLogs.Dequeue ()); + } + LogMessage ($"{diagnosticData.Task}"); + foreach (var t in diagnosticData.Properties) { + LogMessage ($"\t{t.Key}: {t.Value}"); + } + } + + void LogDiagnosticDataError (string errorCode, string error, string file = "") + { + diagnosticData.SetProperty ("deploy.result", "Failed"); + if (!string.IsNullOrEmpty (file)) + diagnosticData.SetProperty ("pii.deploy.file", file); + diagnosticData.SetProperty ("pii.deploy.error", error); + diagnosticData.SetProperty ("deploy.error.code", errorCode); + } + + void SaveDiagnosticData (long ms) + { + JsonSerializerOptions options = new JsonSerializerOptions { + WriteIndented = true + }; + diagnosticData.SetProperty ("deploy.duration.ms", ms); + string newPath = Path.Combine (IntermediateOutputPath, "diagnostics", $"{GetType ().Name.ToLowerInvariant ()}.json"); + File.WriteAllText (newPath, JsonSerializer.Serialize (diagnosticData, options)); + } + + protected string GetFullPath (string dir) => Path.IsPathRooted (dir) ? dir : Path.GetFullPath (Path.Combine (WorkingDirectory, dir)); + + protected bool RaiseRunAsError (string error) + { + if (TryGetRunAsErrorCode (error, out var err)) { + LogDiagnosticDataError (err.code, err.message); + PrintDiagnostics (); + LogCodedError (err.code, err.message, error); + return true; + } + return false; + } + + bool TryGetRunAsErrorCode (string error, out (string error, string code, string message) errTuple) + { + errTuple = (error: "unknown", code: "XA0132", message: error); + foreach (var err in runas_codes) { + if (error.IndexOf (err.error, StringComparison.OrdinalIgnoreCase) >= 0) { + errTuple = err; + return true; + } + } + return false; + } + + string GetErrorCode (Exception ex) + { + switch (ex) { + case IncompatibleCpuAbiException e: + return "ADB0020"; + case RequiresUninstallException e: + return "ADB0030"; + case SdkNotSupportedException e: + return "ADB0040"; + case PackageAlreadyExistsException e: + return "ADB0050"; + case InsufficientSpaceException e: + return "ADB0060"; + case InstallFailedException e: + return "ADB0010"; + default: + return GetErrorCode (ex.Message); + } + } + + static string GetErrorCode (string message) + { + foreach (var errorCode in error_codes) + if (message.IndexOf (errorCode.message, StringComparison.OrdinalIgnoreCase) >= 0) + return errorCode.code; + + return "ADB1000"; + } + + protected static bool IsShellError (string output, string command) + { + if (string.IsNullOrEmpty (output)) { + return false; + } + return output.IndexOf ($"{command}:", StringComparison.OrdinalIgnoreCase) >= 0 || + output.IndexOf ("No such file or directory", StringComparison.OrdinalIgnoreCase) >= 0 || + output.IndexOf ("Permission denied", StringComparison.OrdinalIgnoreCase) >= 0 || + output.IndexOf ("Read-only file system", StringComparison.OrdinalIgnoreCase) >= 0 || + output.IndexOf ("not found", StringComparison.OrdinalIgnoreCase) >= 0; + } + + protected static bool IsMissingDirectoryError (string output) + { + return !string.IsNullOrEmpty (output) && + output.IndexOf ("No such file or directory", StringComparison.OrdinalIgnoreCase) >= 0; + } + + protected struct AdbCommandResult + { + public int ExitCode; + public string Output; + } + + static readonly List<(string error, string code, string message)> runas_codes = new List<(string error, string code, string message)> () { + { (error: "run-as is disabled", code: "XA0131", message: Resources.XA0131_DeveloperModeNotEnabled ) }, + { (error: "Could not set capabilities", code: "XA0131", message: Resources.XA0131_DeveloperModeNotEnabled ) }, + { (error: "unknown", code: "XA0132", message: Resources.XA0132_PackageNotInstalled ) }, + { (error: "Permission denied", code: "XA0133", message: Resources.XA0133_RunAsPermissionDenied ) }, + { (error: "package not debuggable", code: "XA0134", message: Resources.XA0134_RunAsPackageNotDebuggable ) }, + { (error: "package not an application", code: "XA0135", message: Resources.XA0135_RunAsPackageNotAndApplication ) }, + { (error: "has corrupt installation", code: "XA0136", message: Resources.XA0136_RunAsCorruptInstallation ) }, + { (error: "users can run this program", code: "XA0137", message: Resources.XA0137_RunAsOSCorrupt ) }, + { (error: "set SELinux security context", code: "XA0137", message: Resources.XA0137_RunAsOSCorrupt ) }, + { (error: "to package's data directory", code: "XA0137", message: Resources.XA0137_RunAsOSCorrupt ) }, + { (error: "couldn't stat", code: "XA0137", message: Resources.XA0137_RunAsOSCorrupt ) }, + { (error: "has wrong owner", code: "XA0137", message: Resources.XA0137_RunAsOSCorrupt ) }, + { (error: "readable or writable by others", code: "XA0137", message: Resources.XA0137_RunAsOSCorrupt ) }, + { (error: "not a directory", code: "XA0137", message: Resources.XA0137_RunAsOSCorrupt ) }, + { (error: "run-as:", code: "XA0137", message: Resources.XA0137_RunAsOSCorrupt ) }, + }; + + static readonly List<(string code, string message)> error_codes = new List<(string code, string message)> () { + { (code: "ADB0010", message: nameof (InstallFailedException)) }, + { (code: "ADB0020", message: nameof (IncompatibleCpuAbiException)) }, + { (code: "ADB0030", message: nameof (RequiresUninstallException)) }, + { (code: "ADB0040", message: nameof (SdkNotSupportedException)) }, + { (code: "ADB0050", message: nameof (PackageAlreadyExistsException)) }, + { (code: "ADB0060", message: nameof (InsufficientSpaceException)) }, + { (code: "ADB1001", message: "failed to create session") }, + { (code: "ADB1002", message: "failed to finalize session") }, + { (code: "ADB1003", message: "product directory not specified; set $ANDROID_PRODUCT_OUT") }, + { (code: "ADB1004", message: "server didn't ACK") }, + { (code: "ADB1005", message: "server killed by remote request") }, + { (code: "ADB1006", message: "timed out waiting for threads to finish reading from ADB server") }, + { (code: "ADB1007", message: "usage:") }, + { (code: "ADB1008", message: "bulkIn endpoint not assigned") }, + { (code: "ADB1009", message: "bulkOut endpoint not assigned") }, + { (code: "ADB1010", message: "cannot start server on remote host") }, + { (code: "ADB1011", message: "cap_clear_flag(INHERITABLE) failed") }, + { (code: "ADB1012", message: "cap_clear_flag(PEMITTED) failed") }, + { (code: "ADB1013", message: "cap_set_proc() failed") }, + { (code: "ADB1014", message: "Client not connected") }, + { (code: "ADB1015", message: "Could not find device interface") }, + { (code: "ADB1016", message: "Could not set SELinux context") }, + { (code: "ADB1017", message: "Could not start mdnsd") }, + { (code: "ADB1018", message: "could not start server") }, + { (code: "ADB1019", message: "couldn't allocate StdinReadArgs object") }, + { (code: "ADB1020", message: "couldn't create USB matching dictionary") }, + { (code: "ADB1021", message: "daemon started successfully") }, + { (code: "ADB1022", message: "daemon still not running") }, + { (code: "ADB1023", message: "error: no emulator detected") }, + { (code: "ADB1024", message: "error: shell command too long") }, + { (code: "ADB1025", message: "Failed to allocate key") }, + { (code: "ADB1026", message: "failed to allocate memory for ShellProtocol object") }, + { (code: "ADB1027", message: "failed to allocate new subprocess") }, + { (code: "ADB1028", message: "Failed to convert to public key") }, + { (code: "ADB1029", message: "failed to create pipe to report error") }, + { (code: "ADB1030", message: "failed to create run queue notify socketpair") }, + { (code: "ADB1031", message: "failed to empty run queue notify fd") }, + { (code: "ADB1032", message: "failed to encode RSA public key") }, + { (code: "ADB1033", message: "Failed to generate new key") }, + { (code: "ADB1034", message: "failed to get matching services") }, + { (code: "ADB1035", message: "failed to get user home directory") }, + { (code: "ADB1036", message: "Failed to get user key") }, + { (code: "ADB1037", message: "failed to make run queue notify socket nonblocking") }, + { (code: "ADB1038", message: "Failed to read key") }, + { (code: "ADB1039", message: "failed to register libusb hotplug callback") }, + { (code: "ADB1040", message: "failed to start daemon") }, + { (code: "ADB1041", message: "failed to write to run queue notify fd") }, + { (code: "ADB1042", message: "Key must be a null-terminated string") }, + { (code: "ADB1043", message: "Pipe stalled, clearing stall") }, + { (code: "ADB1044", message: "Public key too large to base64 encode") }, + { (code: "ADB1045", message: "reply fd for adb server to client communication not specified") }, + { (code: "ADB1046", message: "run queue notify fd was closed") }, + { (code: "ADB1047", message: "Unable to get interface class, subclass and protocol") }, + { (code: "ADB1048", message: "usb_read interface was null") }, + { (code: "ADB1049", message: "usb_write interface was null") }, + { (code: "ADB1050", message: "cannot fit pipe handle value into 32-bits") }, + { (code: "ADB1051", message: "connect error for create") }, + { (code: "ADB1052", message: "connect error for finalize") }, + { (code: "ADB1053", message: "connect error for write") }, + { (code: "ADB1054", message: "could not open adb service") }, + { (code: "ADB1055", message: "couldn't parse 'wait-for' command") }, + { (code: "ADB1056", message: "CreateFileW 'nul' failed") }, + { (code: "ADB1057", message: "only wrote") }, + { (code: "ADB1058", message: "error response") }, + { (code: "ADB1059", message: "failed to install") }, + { (code: "ADB1060", message: "failed to read block") }, + { (code: "ADB1061", message: "failed to write data") }, + { (code: "ADB1062", message: "invalid reply fd") }, + { (code: "ADB1063", message: "pre-KitKat sideload connection failed") }, + { (code: "ADB1064", message: "doesn't match this client") }, + { (code: "ADB1065", message: "sideload connection failed") }, + { (code: "ADB1066", message: "unable to connect for backup") }, + { (code: "ADB1067", message: "unable to connect for restore") }, + { (code: "ADB1068", message: "unable to connect for") }, + { (code: "ADB1069", message: "unexpected output length for") }, + { (code: "ADB1070", message: "expected 'any', 'local', or 'usb'") }, + { (code: "ADB1071", message: "attempted to close unregistered usb_handle for") }, + { (code: "ADB1072", message: "attempted to reinitialize adb_server_socket_spec") }, + { (code: "ADB1073", message: "cannot connect to daemon at") }, + { (code: "ADB1074", message: "Cannot mkdir") }, + { (code: "ADB1075", message: "Connection banner is too long") }, + { (code: "ADB1076", message: "Could not clear pipe stall both ends") }, + { (code: "ADB1077", message: "Could not install smartsocket listener") }, + { (code: "ADB1078", message: "Could not open interface") }, + { (code: "ADB1079", message: "Could not register mDNS service") }, + { (code: "ADB1080", message: "Couldn't create a device interface") }, + { (code: "ADB1081", message: "Couldn't grab device from interface") }, + { (code: "ADB1082", message: "Couldn't query the interface") }, + { (code: "ADB1083", message: "daemon not running; starting now at") }, + { (code: "ADB1084", message: "destroying fde not created by fdevent_create") }, + { (code: "ADB1085", message: "Encountered mDNS registration error") }, + { (code: "ADB1086", message: "not implemented on Win32") }, + { (code: "ADB1087", message: "could not connect to TCP port") }, + { (code: "ADB1088", message: "no emulator connected") }, + { (code: "ADB1089", message: "only supports allocating a pty") }, + { (code: "ADB1090", message: "failed to connect to socket") }, + { (code: "ADB1091", message: "failed to convert errno") }, + { (code: "ADB1092", message: "failed to initialize libusb") }, + { (code: "ADB1093", message: "Failed to parse key") }, + { (code: "ADB1094", message: "failed to set non-blocking mode for fd") }, + { (code: "ADB1095", message: "failed to start subprocess management thread") }, + { (code: "ADB1096", message: "failed to start subprocess") }, + { (code: "ADB1097", message: "FindDeviceInterface - could not get pipe properties") }, + { (code: "ADB1098", message: "Invalid base64 key") }, + { (code: "ADB1099", message: "Key too long") }, + { (code: "ADB1100", message: "No ':' found in shell service arguments") }, + { (code: "ADB1101", message: "observed inotify event for unmonitored path") }, + { (code: "ADB1102", message: "packet data length doesn't match payload") }, + { (code: "ADB1103", message: "Unable to create a device plug-in") }, + { (code: "ADB1104", message: "Unable to create an interface plug-in") }, + { (code: "ADB1105", message: "Unable to get number of endpoints") }, + { (code: "ADB1106", message: "unexpected type for") }, + { (code: "ADB1107", message: "Unknown socket type") }, + { (code: "ADB1108", message: "Unknown trace flag") }, + { (code: "ADB1109", message: "usb_read failed with status") }, + { (code: "ADB1110", message: "usb_write failed with status") }, + { (code: "ADB1111", message: "adb_socket_accept: failed to allocate accepted socket") }, + { (code: "ADB1112", message: "cannot create service socket pair") }, + { (code: "ADB1113", message: "cannot create socket pair") }, + { (code: "ADB1114", message: "Error generating token") }, + { (code: "ADB1115", message: "Error getting user key filename") }, + { (code: "ADB1116", message: "Failed to accept") }, + { (code: "ADB1117", message: "failed to create inotify fd") }, + { (code: "ADB1118", message: "Failed to get adbd socket") }, + { (code: "ADB1119", message: "failed to shutdown writes to FD") }, + { (code: "ADB1120", message: "Failed to write PK") }, + { (code: "ADB1121", message: "failed to write the exit code packet") }, + { (code: "ADB1122", message: "read of inotify event failed") }, + { (code: "ADB1123", message: "remote usb: 1 - write terminated") }, + { (code: "ADB1124", message: "remote usb: 2 - write terminated") }, + { (code: "ADB1125", message: "remote usb: read terminated (message)") }, + { (code: "ADB1126", message: "remote usb: terminated (data)") }, + { (code: "ADB1127", message: "select failed, closing subprocess pipes") }, + { (code: "ADB1128", message: "backup unable to create file") }, + { (code: "ADB1129", message: "cannot create thread") }, + { (code: "ADB1130", message: "cannot get executable path") }, + { (code: "ADB1131", message: "cannot make handle") }, + { (code: "ADB1132", message: "CreatePipe failed") }, + { (code: "ADB1133", message: "CreateProcessW failed") }, + { (code: "ADB1134", message: "error while reading for") }, + { (code: "ADB1135", message: "execl returned") }, + { (code: "ADB1136", message: "failed to duplicate file descriptor for") }, + { (code: "ADB1137", message: "failed to get file descriptor for") }, + { (code: "ADB1138", message: "failed to open duplicate stream for") }, + { (code: "ADB1139", message: "failed to open file") }, + { (code: "ADB1140", message: "failed to read command") }, + { (code: "ADB1141", message: "failed to read data from") }, + { (code: "ADB1142", message: "failed to read from") }, + { (code: "ADB1143", message: "failed to read package block") }, + { (code: "ADB1144", message: "failed to seek to package block") }, + { (code: "ADB1145", message: "failed to set binary mode for duplicate of") }, + { (code: "ADB1146", message: "failed to stat file") }, + { (code: "ADB1147", message: "failed to stat") }, + { (code: "ADB1148", message: "failed to unbuffer") }, + { (code: "ADB1149", message: "adb_socket_accept: accept on fd") }, + { (code: "ADB1150", message: "unable to open file") }, + { (code: "ADB1151", message: "unexpected result waiting for threads") }, + { (code: "ADB1152", message: "aio: got error event on") }, + { (code: "ADB1153", message: "aio: got error submitting") }, + { (code: "ADB1154", message: "aio: got error waiting") }, + { (code: "ADB1155", message: "cannot open bulk-in endpoint") }, + { (code: "ADB1156", message: "cannot open bulk-out endpoint") }, + { (code: "ADB1157", message: "cannot open control endpoint") }, + { (code: "ADB1158", message: "Can't load") }, + { (code: "ADB1159", message: "could not read ok from ADB Server") }, + { (code: "ADB1160", message: "couldn't allocate state_info") }, + { (code: "ADB1161", message: "Couldn't read") }, + { (code: "ADB1162", message: "cannot write to emulator") }, + { (code: "ADB1163", message: "error reading output FD") }, + { (code: "ADB1164", message: "error reading protocol FD") }, + { (code: "ADB1165", message: "error reading stdin FD") }, + { (code: "ADB1166", message: "write failure during connection") }, + { (code: "ADB1167", message: "failed to fcntl(F_GETFL) for fd") }, + { (code: "ADB1168", message: "failed to fcntl(F_SETFL) for fd") }, + { (code: "ADB1169", message: "failed to inotify_add_watch on path") }, + { (code: "ADB1170", message: "Failed to listen on") }, + { (code: "ADB1171", message: "failed to open directory") }, + { (code: "ADB1172", message: "Failed to write public key to") }, + { (code: "ADB1173", message: "failure closing FD") }, + { (code: "ADB1174", message: "pipe failed in launch_server") }, + { (code: "ADB1175", message: "poll() }, ret =") }, + { (code: "ADB1176", message: "remote usb: read overflow") }, + { (code: "ADB1177", message: "received framework auth socket connection again") }, + { (code: "ADB1178", message: "failed to claim adb interface for device") }, + { (code: "ADB1179", message: "failed to clear halt on device") }, + { (code: "ADB1180", message: "failed to get active config descriptor for device at") }, + { (code: "ADB1181", message: "failed to get device descriptor for device at") }, + { (code: "ADB1182", message: "failed to get serial from device at") }, + { (code: "ADB1183", message: "failed to open usb device at") }, + { (code: "ADB1184", message: "failed to set interface alt setting for device") }, + { (code: "ADB1185", message: "failed to submit zero-length write") }, + { (code: "ADB1186", message: "failed to submit") }, + { (code: "ADB1187", message: "Ignoring unknown shell service argument") }, + { (code: "ADB1188", message: "transfer failed:") }, + { (code: "ADB1189", message: "received empty serial from device at") }, + { (code: "ADB1190", message: "refusing to recurse into directory") }, + { (code: "ADB1191", message: "unmonitored event for") }, + { (code: "ADB1192", message: "Failed to open") }, + { (code: "ADB1193", message: "failed to write") }, + }; + + static readonly Regex AdbPushSummaryRegex = new Regex (@"(?\d+) files? pushed, (?\d+) skipped", RegexOptions.Compiled); + } +} diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy3.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy3.cs new file mode 100644 index 00000000000..6e405e31a91 --- /dev/null +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy3.cs @@ -0,0 +1,259 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading.Tasks; + +namespace Xamarin.Android.Tasks +{ + public class FastDeploy3 : FastDeploy2 + { + const string RemoteDataLocalTmpRoot = "/data/local/tmp/fastdeploy3"; + const int CopyBatchSize = 25; + const int StaleFileRemovalBatchSize = 100; + + public override string TaskPrefix => "FD3"; + + protected override string RemoteStagingRoot => RemoteDataLocalTmpRoot; + + protected override string GetLocalStagingDirectory () + { + return Path.Combine (GetAndroidProductOutDirectory (), "data", "local", "tmp", "fastdeploy3", PackageName, GetUserId ()); + } + + protected override string GetRemoteAdbPushStagingPath () + { + return $"{RemoteDataLocalTmpRoot}/{PackageName}/{GetUserId ()}"; + } + + protected override async Task DeployFastDevFilesWithAdbPush (string overridePath) + { + string stagingDirectory = GetLocalStagingDirectory (); + var phase = Stopwatch.StartNew (); + var stagedFiles = PrepareAdbPushStagingDirectory (stagingDirectory); + SetDiagnosticElapsed ("deploy.fastdeploy2.local.stage.ms", phase); + if (stagedFiles.Count == 0) { + LogDiagnostic ("No FastDev files were staged for adb sync deployment."); + return true; + } + + string remoteStagingPath = GetRemoteAdbPushStagingPath (); + phase.Restart (); + var mkdirResult = await RunAdbCommand ("shell", "mkdir", "-p", remoteStagingPath); + string output = mkdirResult.Output; + SetDiagnosticElapsed ("deploy.fastdeploy2.remote.mkdir.ms", phase); + if (mkdirResult.ExitCode != 0 || IsShellError (output, "mkdir")) { + LogFastDeploy2Error ("XA0129", output, remoteStagingPath); + return false; + } + + if (!await RemoveStaleRemoteStagingFiles (remoteStagingPath, stagedFiles)) { + return false; + } + + phase.Restart (); + var syncChangedFiles = await GetChangedFilesFromSyncList (remoteStagingPath); + SetDiagnosticElapsed ("deploy.fastdeploy3.sync.list.ms", phase); + if (syncChangedFiles == null) { + return false; + } + SetDiagnosticProperty ("deploy.fastdeploy3.sync.list.files", syncChangedFiles.Count); + + phase.Restart (); + if (!await UploadStagingDirectory (stagingDirectory, remoteStagingPath)) { + return false; + } + SetDiagnosticElapsed ("deploy.fastdeploy2.upload.ms", phase); + + phase.Restart (); + var overrideFiles = await GetOverrideFileList (overridePath); + SetDiagnosticElapsed ("deploy.fastdeploy2.override.stat.ms", phase); + SetDiagnosticElapsed ("deploy.fastdeploy3.override.list.ms", phase); + if (overrideFiles == null) { + return false; + } + + phase.Restart (); + var changedFiles = new HashSet (syncChangedFiles, StringComparer.Ordinal); + int missingFiles = 0; + foreach (var file in stagedFiles) { + if (!overrideFiles.Contains (file) && changedFiles.Add (file)) { + missingFiles++; + } + } + SetDiagnosticElapsed ("deploy.fastdeploy2.compare.ms", phase); + SetDiagnosticProperty ("deploy.fastdeploy3.missing.files", missingFiles); + + if (!await RemoveStaleOverrideFiles (overridePath, stagedFiles, overrideFiles)) { + return false; + } + + return await CopyChangedFiles (remoteStagingPath, overridePath, changedFiles); + } + + protected override async Task UploadStagingDirectory (string stagingDirectory, string remoteStagingPath) + { + var args = new List { "sync" }; + if (!string.IsNullOrEmpty (AdbPushCompressionAlgorithm)) { + args.Add ("-z"); + args.Add (AdbPushCompressionAlgorithm); + } + args.Add ("data"); + + var environmentVariables = new Dictionary { + { "ANDROID_PRODUCT_OUT", GetAndroidProductOutDirectory () }, + }; + var result = await RunAdbCommand (args.ToArray (), environmentVariables); + if (result.ExitCode != 0) { + LogFastDeploy2Error ("XA0129", result.Output, stagingDirectory); + return false; + } + SetAdbPushFileCounts (result.Output); + LogDiagnostic (result.Output); + return true; + } + + async Task> GetChangedFilesFromSyncList (string remoteStagingPath) + { + var args = new [] { "sync", "-l", "data" }; + var environmentVariables = new Dictionary { + { "ANDROID_PRODUCT_OUT", GetAndroidProductOutDirectory () }, + }; + var result = await RunAdbCommand (args, environmentVariables); + if (result.ExitCode != 0) { + LogFastDeploy2Error ("XA0129", result.Output, GetAndroidProductOutDirectory ()); + return null; + } + + var changedFiles = new HashSet (StringComparer.Ordinal); + string prefix = remoteStagingPath.TrimEnd ('/') + "/"; + foreach (string line in result.Output.Split (new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) { + if (!line.StartsWith ("would push:", StringComparison.Ordinal)) { + continue; + } + + int index = line.LastIndexOf (" -> ", StringComparison.Ordinal); + if (index < 0) { + LogDebugMessage ($"Ignoring adb sync -l line '{line}'. Line is incorrectly formatted."); + continue; + } + + string remoteFile = line.Substring (index + 4).Trim (); + if (!remoteFile.StartsWith (prefix, StringComparison.Ordinal)) { + LogDebugMessage ($"Ignoring adb sync -l line '{line}'. Path is outside '{remoteStagingPath}'."); + continue; + } + changedFiles.Add (remoteFile.Substring (prefix.Length)); + } + LogDiagnostic ($"FastDeploy3 adb sync -l listed {changedFiles.Count} changed files."); + return changedFiles; + } + + async Task> GetOverrideFileList (string overridePath) + { + string output = await RunAs ("find", overridePath, "-type", "f"); + if (RaiseRunAsError (output)) { + return null; + } + if (IsMissingDirectoryError (output)) { + return new HashSet (StringComparer.Ordinal); + } + if (IsShellError (output, "find")) { + LogFastDeploy2Error ("XA0129", output, overridePath); + return null; + } + + var files = new HashSet (StringComparer.Ordinal); + string prefix = overridePath.TrimEnd ('/') + "/"; + foreach (string line in output.Split (new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) { + string remoteFile = line.Trim (); + if (!remoteFile.StartsWith (prefix, StringComparison.Ordinal)) { + LogDebugMessage ($"Ignoring override file entry '{line}'. Path is outside '{overridePath}'."); + continue; + } + files.Add (remoteFile.Substring (prefix.Length)); + } + return files; + } + + async Task RemoveStaleOverrideFiles (string overridePath, HashSet stagedFiles, HashSet overrideFiles) + { + var phase = Stopwatch.StartNew (); + var staleFiles = new List (); + foreach (var file in overrideFiles) { + if (!stagedFiles.Contains (file)) { + staleFiles.Add ($"{overridePath}/{file}"); + } + } + + LogDiagnostic ($"FastDeploy3 removing {staleFiles.Count} stale override files."); + SetDiagnosticProperty ("deploy.fastdeploy2.stale.files", staleFiles.Count); + for (int i = 0; i < staleFiles.Count; i += StaleFileRemovalBatchSize) { + var args = new List { "rm", "-f" }; + args.AddRange (staleFiles.Skip (i).Take (StaleFileRemovalBatchSize)); + string output = await RunAs (args.ToArray ()); + if (RaiseRunAsError (output) || IsShellError (output, "rm")) { + LogFastDeploy2Error ("XA0129", output, overridePath); + return false; + } + } + SetDiagnosticElapsed ("deploy.fastdeploy2.stale.remove.ms", phase); + return true; + } + + async Task CopyChangedFiles (string remoteStagingPath, string overridePath, HashSet changedFiles) + { + LogDiagnostic ($"FastDeploy3 copying {changedFiles.Count} changed override files."); + SetDiagnosticProperty ("deploy.fastdeploy2.changed.files", changedFiles.Count); + var filesByDirectory = new Dictionary> (StringComparer.Ordinal); + foreach (string file in changedFiles) { + string directory = Path.GetDirectoryName (file)?.Replace ("\\", "/") ?? ""; + if (!filesByDirectory.TryGetValue (directory, out List files)) { + files = new List (); + filesByDirectory.Add (directory, files); + } + files.Add (file); + } + + var phase = Stopwatch.StartNew (); + foreach (var group in filesByDirectory) { + string targetDirectory = string.IsNullOrEmpty (group.Key) ? overridePath : $"{overridePath}/{group.Key}"; + phase.Restart (); + string output = await RunAs ("mkdir", "-p", targetDirectory); + AddDiagnosticElapsed ("deploy.fastdeploy2.override.mkdir.ms", phase); + if (RaiseRunAsError (output) || IsShellError (output, "mkdir")) { + LogFastDeploy2Error ("XA0129", output, targetDirectory); + return false; + } + + for (int i = 0; i < group.Value.Count; i += CopyBatchSize) { + var args = new List { "cp", "-p" }; + foreach (string file in group.Value.Skip (i).Take (CopyBatchSize)) { + args.Add ($"{remoteStagingPath}/{file}"); + } + args.Add (targetDirectory); + phase.Restart (); + output = await RunAs (args.ToArray ()); + AddDiagnosticElapsed ("deploy.fastdeploy2.override.copy.ms", phase); + if (RaiseRunAsError (output) || IsShellError (output, "cp")) { + LogFastDeploy2Error ("XA0129", output, targetDirectory); + return false; + } + } + } + + return true; + } + + string GetAndroidProductOutDirectory () + { + return Path.Combine (GetFullPath (IntermediateOutputPath), "fastdeploy3-product-out"); + } + + string GetUserId () + { + return string.IsNullOrEmpty (UserID) ? "0" : UserID; + } + } +} diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets index dc270a09cda..d5f457ded45 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets @@ -23,6 +23,8 @@ Copyright (C) 2016 Xamarin. All rights reserved. + + @@ -321,7 +323,12 @@ Copyright (C) 2016 Xamarin. All rights reserved. <_ReInstall Condition=" '$(_ReInstall)' == '' ">False <_AndroidIsTestOnlyPackage Condition=" '$(_AndroidIsTestOnlyPackage)' == '' ">False <_FastDeploymentDiagnosticLogging Condition=" '$(_FastDeploymentDiagnosticLogging)' == '' ">False + <_AndroidFastDevStrategy Condition=" '$(_AndroidFastDevStrategy)' == '' ">FastDeploy + any + @@ -331,6 +338,7 @@ Copyright (C) 2016 Xamarin. All rights reserved. <_EnvironmentFiles Include="@(AndroidEnvironment);@(LibraryEnvironments)" /> + + Date: Wed, 17 Jun 2026 23:59:13 +0200 Subject: [PATCH 02/12] Add FastDeploy orchestration timing diagnostics Capture non-transfer phase timings for FastDeploy2 and FastDeploy3 so the constant overhead can be broken down separately from staging, upload, stat, and copy work. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tasks/FastDeploy2.cs | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs index f4ab215bddd..a0d496873d9 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs @@ -128,6 +128,13 @@ class DiagnosticData { { "deploy.fastdeploy3.sync.list.files", "" }, { "deploy.fastdeploy3.override.list.ms", "" }, { "deploy.fastdeploy3.missing.files", "" }, + { "deploy.orchestration.ensure-properties.ms", "" }, + { "deploy.orchestration.property-checks.ms", "" }, + { "deploy.orchestration.package-check.ms", "" }, + { "deploy.orchestration.package-timestamp.ms", "" }, + { "deploy.orchestration.install.ms", "" }, + { "deploy.orchestration.terminate.ms", "" }, + { "deploy.orchestration.empty-check.ms", "" }, { "pii.deploy.error", "" }, { "pii.deploy.file", "" }, }; @@ -216,8 +223,11 @@ public async override Task RunTaskAsync () async Task RunInstall () { + var phase = Stopwatch.StartNew (); await Device.EnsureProperties (CancellationToken).ConfigureAwait (false); + SetDiagnosticElapsed ("deploy.orchestration.ensure-properties.ms", phase); + phase.Restart (); diagnosticData.SetProperty ("target.prop.ro.product.build.version.sdk", Device.Properties?.BuildVersionSdk); diagnosticData.SetProperty ("target.prop.ro.product.cpu.abilist", string.Join (";", Device.Properties?.ProductCpuAbiList ?? Array.Empty ())); diagnosticData.SetProperty ("target.prop.ro.product.cpu.abi", PrimaryCpuAbi); @@ -235,8 +245,11 @@ async Task RunInstall () LogFastDeploy2Error ("XA0131", Resources.XA0131_DeveloperModeNotEnabled); return; } + SetDiagnosticElapsed ("deploy.orchestration.property-checks.ms", phase); + phase.Restart (); await CheckAppInstalledAndDebuggable (PackageName); + SetDiagnosticElapsed ("deploy.orchestration.package-check.ms", phase); if (EmbedAssembliesIntoApk) { await RemoveOverrideDirectory (); @@ -246,17 +259,26 @@ async Task RunInstall () await Device.UninstallPackage (PackageName, PreserveUserData, CancellationToken); } - if (!string.IsNullOrEmpty (PackageFile) && - (packageInfo.InternalPath.IndexOf ("unknown", StringComparison.OrdinalIgnoreCase) >= 0 || ReInstall || IsPackageFileOutOfDate ())) { + phase.Restart (); + bool packageFileOutOfDate = !string.IsNullOrEmpty (PackageFile) && + (packageInfo.InternalPath.IndexOf ("unknown", StringComparison.OrdinalIgnoreCase) >= 0 || ReInstall || IsPackageFileOutOfDate ()); + SetDiagnosticElapsed ("deploy.orchestration.package-timestamp.ms", phase); + + if (packageFileOutOfDate) { try { + phase.Restart (); await InstallPackage (); + AddDiagnosticElapsed ("deploy.orchestration.install.ms", phase); } catch (Exception ex) { + AddDiagnosticElapsed ("deploy.orchestration.install.ms", phase); LogFastDeploy2Error (GetErrorCode (ex), ex.ToString ()); return; } if (!EmbedAssembliesIntoApk && packageInfo.InternalPath.IndexOf ("unknown", StringComparison.OrdinalIgnoreCase) >= 0) { packageInfo.InternalPath = null; + phase.Restart (); await CheckAppInstalledAndDebuggable (PackageName); + AddDiagnosticElapsed ("deploy.orchestration.package-check.ms", phase); if (RaiseRunAsError (packageInfo.InternalPath)) { return; } @@ -266,11 +288,16 @@ async Task RunInstall () if (EmbedAssembliesIntoApk) return; + phase.Restart (); if ((FastDevFiles?.Length ?? 0) == 0 && (EnvironmentFiles?.Length ?? 0) == 0) { + SetDiagnosticElapsed ("deploy.orchestration.empty-check.ms", phase); return; } + SetDiagnosticElapsed ("deploy.orchestration.empty-check.ms", phase); + phase.Restart (); await TerminateApp (); + SetDiagnosticElapsed ("deploy.orchestration.terminate.ms", phase); await DeployFastDevFilesWithAdbPush (OverrideFullPath); } From 0e5fa52ce2821aa6aa99f47f0941d20d6d02a6e7 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Thu, 18 Jun 2026 08:59:16 +0200 Subject: [PATCH 03/12] Add fine-grained FastDeploy orchestration timings Break orchestration timing down into package checks, package install, process lookup/kill, and execute/setup substeps so the fixed costs can be measured independently from file transfer phases. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tasks/FastDeploy2.cs | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs index a0d496873d9..7d70b32ea57 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs @@ -135,6 +135,25 @@ class DiagnosticData { { "deploy.orchestration.install.ms", "" }, { "deploy.orchestration.terminate.ms", "" }, { "deploy.orchestration.empty-check.ms", "" }, + { "deploy.execute.parse-target.ms", "" }, + { "deploy.execute.no-abi-check.ms", "" }, + { "deploy.execute.upload-flag-stat.ms", "" }, + { "deploy.execute.task-cache.ms", "" }, + { "deploy.orchestration.property-capture.ms", "" }, + { "deploy.orchestration.redirect-stdio-check.ms", "" }, + { "deploy.orchestration.run-as-disabled-check.ms", "" }, + { "deploy.orchestration.package-check.ensure-user.ms", "" }, + { "deploy.orchestration.package-check.run-as-pwd.ms", "" }, + { "deploy.orchestration.package-check.readlink.ms", "" }, + { "deploy.orchestration.package-check.system-app.ms", "" }, + { "deploy.orchestration.package-check.evaluate.ms", "" }, + { "deploy.orchestration.package-timestamp.path-stat.ms", "" }, + { "deploy.orchestration.install.push-install.ms", "" }, + { "deploy.orchestration.install.retry-delete.ms", "" }, + { "deploy.orchestration.install.retry-uninstall.ms", "" }, + { "deploy.orchestration.install.retry-reinstall.ms", "" }, + { "deploy.orchestration.terminate.get-pid.ms", "" }, + { "deploy.orchestration.terminate.kill.ms", "" }, { "pii.deploy.error", "" }, { "pii.deploy.file", "" }, }; @@ -172,24 +191,32 @@ void DebugHandler (string task, string message) public override bool Execute () { + var phase = Stopwatch.StartNew (); Device = AndroidHelper.ParseTarget (AdbTarget, LogMessage, LogCodedError, logErrors: true, engine4: BuildEngine4); + SetDiagnosticElapsed ("deploy.execute.parse-target.ms", phase); if (Device == null) { PrintDiagnostics (); return false; } LogMessage ($"Found device: {Device.ID}"); + phase.Restart (); if (string.IsNullOrEmpty (PrimaryCpuAbi) && !EmbedAssembliesIntoApk) { + SetDiagnosticElapsed ("deploy.execute.no-abi-check.ms", phase); PrintDiagnostics (); LogCodedError ("XA0010", Resources.XA0010_NoAbi, Device.ID); return false; } + SetDiagnosticElapsed ("deploy.execute.no-abi-check.ms", phase); + phase.Restart (); var flagFilePath = GetFullPath (UploadFlagFile); lastUpload = File.GetLastWriteTimeUtc (flagFilePath); LogDiagnostic ($"LastWriteTime of `{flagFilePath}`: {lastUpload}"); diagnosticData.Task = GetType ().Name; + SetDiagnosticElapsed ("deploy.execute.upload-flag-stat.ms", phase); + phase.Restart (); var lifetime = RegisteredTaskObjectLifetime.AppDomain; var key = ProjectSpecificTaskObjectKey ($"{Device.ID}_{PackageName}_{GetType ().Name}"); if (!File.Exists (UploadFlagFile)) { @@ -197,6 +224,7 @@ public override bool Execute () } else { packageInfo = BuildEngine4.GetRegisteredTaskObjectAssemblyLocal (key, lifetime) ?? new PackageInfo (); } + SetDiagnosticElapsed ("deploy.execute.task-cache.ms", phase); AndroidLogger.Debug += DebugHandler; try { @@ -233,14 +261,19 @@ async Task RunInstall () diagnosticData.SetProperty ("target.prop.ro.product.cpu.abi", PrimaryCpuAbi); diagnosticData.SetProperty ("target.prop.ro.product.manufacturer", Device.Properties?.ProductManufacturer); diagnosticData.SetProperty ("target.prop.ro.product.model", Device.Properties?.ProductModel); + SetDiagnosticElapsed ("deploy.orchestration.property-capture.ms", phase); + phase.Restart (); string redirectStdio = Device.Properties.Get ("log.redirect-stdio"); + SetDiagnosticElapsed ("deploy.orchestration.redirect-stdio-check.ms", phase); if (redirectStdio != null && string.Equals ("true", redirectStdio.Trim (), StringComparison.OrdinalIgnoreCase)) { LogFastDeploy2Error ("XA0128", Resources.XA0128_RedirectStdioIsEnabled); return; } + phase.Restart (); string runAsDisabled = Device.Properties.Get ("ro.boot.disable_runas"); + SetDiagnosticElapsed ("deploy.orchestration.run-as-disabled-check.ms", phase); if (runAsDisabled != null && string.Equals ("true", runAsDisabled.Trim (), StringComparison.OrdinalIgnoreCase)) { LogFastDeploy2Error ("XA0131", Resources.XA0131_DeveloperModeNotEnabled); return; @@ -303,21 +336,30 @@ async Task RunInstall () bool IsPackageFileOutOfDate () { + var phase = Stopwatch.StartNew (); var packageFile = GetFullPath (PackageFile); var lastPackage = File.GetLastWriteTimeUtc (packageFile); LogDiagnostic ($"LastWriteTime of `{packageFile}`: {lastPackage}"); + SetDiagnosticElapsed ("deploy.orchestration.package-timestamp.path-stat.ms", phase); return lastUpload < lastPackage; } async Task CheckAppInstalledAndDebuggable (string packageName) { + var phase = Stopwatch.StartNew (); packageInfo.UserId = UserID; packageInfo.PackageName = packageName; await EnsureUserIsRunning (); + SetDiagnosticElapsed ("deploy.orchestration.package-check.ensure-user.ms", phase); + phase.Restart (); packageInfo.InternalPath = packageInfo.InternalPath ?? await RunAs ("pwd"); + SetDiagnosticElapsed ("deploy.orchestration.package-check.run-as-pwd.ms", phase); if (packageInfo.InternalPath.IndexOf ("Permission denied", StringComparison.OrdinalIgnoreCase) >= 0) { + phase.Restart (); packageInfo.InternalPath = await RunAs ("readlink", "-f", "."); + SetDiagnosticElapsed ("deploy.orchestration.package-check.readlink.ms", phase); } + phase.Restart (); if (packageInfo.InternalPath.IndexOf ("not an application", StringComparison.OrdinalIgnoreCase) >= 0) { LogDiagnostic ($"Package {packageInfo.PackageName} is a system application."); packageInfo.IsSystemApplication = true; @@ -326,21 +368,25 @@ async Task CheckAppInstalledAndDebuggable (string packageName) packageInfo.AdbIsRoot = whoami.Trim () == "root"; LogDiagnostic ($"using {(packageInfo.AdbIsRoot ? "root" : $"su {packageInfo.UserId}")} to install fast deployment files."); packageInfo.InternalPath = $"/data/user/{(packageInfo.UserId ?? "0")}/{packageInfo.PackageName}"; + SetDiagnosticElapsed ("deploy.orchestration.package-check.system-app.ms", phase); return; } if (packageInfo.InternalPath.IndexOf ("not debuggable", StringComparison.OrdinalIgnoreCase) >= 0) { LogDiagnostic ($"Package {packageInfo.PackageName} was not debuggable. Forcing ReInstall"); ReInstall = true; + SetDiagnosticElapsed ("deploy.orchestration.package-check.evaluate.ms", phase); return; } if (packageInfo.InternalPath.IndexOf ("unknown", StringComparison.OrdinalIgnoreCase) >= 0) { LogDiagnostic ($"Package {packageInfo.PackageName} was not installed."); + SetDiagnosticElapsed ("deploy.orchestration.package-check.evaluate.ms", phase); return; } if (packageInfo.InternalPath.IndexOf ("Permission denied", StringComparison.OrdinalIgnoreCase) >= 0) { LogDiagnostic ("run-as not supported on this device."); diagnosticData.SetProperty ("deploy.supports.fastdev", value: false); } + SetDiagnosticElapsed ("deploy.orchestration.package-check.evaluate.ms", phase); } async Task EnsureUserIsRunning () @@ -358,6 +404,7 @@ async Task InstallPackage () { LogDebugMessage ($"Installing Package {PackageName}"); try { + var phase = Stopwatch.StartNew (); await Device.PushAndInstallPackageAsync (new PushAndInstallCommand { ApkFile = PackageFile, PackageName = PackageName, @@ -365,6 +412,7 @@ await Device.PushAndInstallPackageAsync (new PushAndInstallCommand { User = UserID, TestOnly = IsTestOnly, }, token: CancellationToken); + SetDiagnosticElapsed ("deploy.orchestration.install.push-install.ms", phase); LogDebugMessage ($"Installed Package {PackageName}."); } catch (Exception exception) { var ex = exception; @@ -391,21 +439,27 @@ async Task ShouldThrowIfPackageInstallFailed (PackageAlreadyExistsExceptio return false; LogDebugMessage (string.Format ("Package '{0}' already exists. Retrying...", PackageName)); + var phase = Stopwatch.StartNew (); try { await Device.DeleteFile (e.PackageFile, true, CancellationToken); } catch { } + SetDiagnosticElapsed ("deploy.orchestration.install.retry-delete.ms", phase); bool preserveData = !(e is RequiresUninstallException); LogDebugMessage (string.Format ("Forcing complete uninstall of '{0}'... Preserving Data: {1}", PackageName, preserveData)); var uninstallCommand = new PmUninstallCommand () { PackageName = PackageName, User = UserID, PreserveData = preserveData }; + phase.Restart (); await Device.UninstallPackage (uninstallCommand, cancellationToken: CancellationToken); + SetDiagnosticElapsed ("deploy.orchestration.install.retry-uninstall.ms", phase); LogDebugMessage (string.Format ("Installing '{0}'...", PackageName)); + phase.Restart (); await Device.PushAndInstallPackageAsync (new PushAndInstallCommand { ApkFile = PackageFile, PackageName = PackageName, ReInstall = false, User = UserID }, token: CancellationToken); + SetDiagnosticElapsed ("deploy.orchestration.install.retry-reinstall.ms", phase); return false; } @@ -416,13 +470,17 @@ async Task RemoveOverrideDirectory () async Task TerminateApp () { + var phase = Stopwatch.StartNew (); var pid = await Device.GetProcessId (PackageName, CancellationToken); + SetDiagnosticElapsed ("deploy.orchestration.terminate.get-pid.ms", phase); if (pid == 0) { LogDebugMessage ($"{PackageName} was not running, skipping kill"); return; } LogDebugMessage ($"Terminating {PackageName}..."); + phase.Restart (); await Device.KillProcessAndWaitForExit (PackageName, CancellationToken); + SetDiagnosticElapsed ("deploy.orchestration.terminate.kill.ms", phase); LogDebugMessage ($"{PackageName} Terminated."); } From 244209ff7d7d32f3dab21aa822daf9a45e9f8375 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Thu, 18 Jun 2026 09:34:31 +0200 Subject: [PATCH 04/12] Add direct-push FastDeploy4 strategy Introduce FastDeploy4 with direct original-file adb push support and selectable SingleFile/Bulk push modes so local staging costs can be evaluated independently. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tasks/FastDeploy2.cs | 4 +- .../Tasks/FastDeploy4.cs | 489 ++++++++++++++++++ .../Xamarin.Android.Common.Debugging.targets | 34 +- 3 files changed, 523 insertions(+), 4 deletions(-) create mode 100644 src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs index 7d70b32ea57..db8214fe9b2 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs @@ -179,7 +179,7 @@ internal void SetProperty (string key, string value) } } - class RemoteFileInfo { + protected class RemoteFileInfo { public long Size { get; set; } public long ModifiedTime { get; set; } } @@ -691,7 +691,7 @@ protected async Task RemoveStaleRemoteStagingFiles (string remoteStagingPa return true; } - async Task> GetRemoteFileData (string rootPath, bool runAs) + protected async Task> GetRemoteFileData (string rootPath, bool runAs) { string output; if (runAs) { diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs new file mode 100644 index 00000000000..e3ce3ca78b3 --- /dev/null +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs @@ -0,0 +1,489 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using Microsoft.Android.Build.Tasks; + +namespace Xamarin.Android.Tasks +{ + public class FastDeploy4 : FastDeploy2 + { + const string RemoteStagingRootPath = "/tmp/fastdeploy4"; + const int CopyBatchSize = 25; + const int RemoveBatchSize = 100; + const int MaxAdbCommandLength = 4096; + + public override string TaskPrefix => "FD4"; + + public string PushMode { get; set; } = "SingleFile"; + + protected override string RemoteStagingRoot => RemoteStagingRootPath; + + protected override async Task DeployFastDevFilesWithAdbPush (string overridePath) + { + string remoteStagingPath = GetRemoteAdbPushStagingPath (); + var phase = Stopwatch.StartNew (); + var files = PrepareDirectPushFiles (); + var expectedFiles = new HashSet (files.Select (file => file.RelativePath), StringComparer.Ordinal); + SetDiagnosticElapsed ("deploy.fastdeploy2.local.stage.ms", phase); + if (files.Count == 0) { + LogDiagnostic ("No FastDev files were prepared for direct adb push deployment."); + return true; + } + + phase.Restart (); + if (!await CreateRemoteStagingDirectories (remoteStagingPath, files)) { + return false; + } + SetDiagnosticElapsed ("deploy.fastdeploy2.remote.mkdir.ms", phase); + + if (!await RemoveStaleRemoteStagingFiles (remoteStagingPath, expectedFiles)) { + return false; + } + + var pushMode = PushMode ?? ""; + HashSet changedFiles; + if (string.Equals (pushMode, "SingleFile", StringComparison.OrdinalIgnoreCase)) { + phase.Restart (); + changedFiles = await PushFilesOneByOne (remoteStagingPath, files); + SetDiagnosticElapsed ("deploy.fastdeploy2.upload.ms", phase); + } else if (string.Equals (pushMode, "Bulk", StringComparison.OrdinalIgnoreCase)) { + phase.Restart (); + if (!await PushFilesInBulk (remoteStagingPath, files)) { + return false; + } + SetDiagnosticElapsed ("deploy.fastdeploy2.upload.ms", phase); + + phase.Restart (); + var stagedFileData = await GetRemoteFileData (remoteStagingPath, runAs: false); + SetDiagnosticElapsed ("deploy.fastdeploy2.staging.stat.ms", phase); + if (stagedFileData == null) { + return false; + } + + phase.Restart (); + var overrideFileData = await GetRemoteFileData (overridePath, runAs: true); + SetDiagnosticElapsed ("deploy.fastdeploy2.override.stat.ms", phase); + if (overrideFileData == null) { + return false; + } + + if (!await RemoveStaleOverrideFiles (overridePath, stagedFileData.Keys, overrideFileData.Keys)) { + return false; + } + + phase.Restart (); + changedFiles = GetChangedFiles (stagedFileData, overrideFileData); + SetDiagnosticElapsed ("deploy.fastdeploy2.compare.ms", phase); + } else { + LogFastDeploy2Error ("XA0129", $"Invalid FastDeploy4 PushMode '{PushMode}'. Supported values are 'SingleFile' and 'Bulk'.", PushMode); + return false; + } + + SetDiagnosticProperty ("deploy.fastdeploy4.push.mode", pushMode); + + phase.Restart (); + var overrideFiles = await GetOverrideFileList (overridePath); + SetDiagnosticElapsed ("deploy.fastdeploy3.override.list.ms", phase); + if (overrideFiles == null) { + return false; + } + + phase.Restart (); + int missingFiles = 0; + foreach (string file in expectedFiles) { + if (!overrideFiles.Contains (file) && changedFiles.Add (file)) { + missingFiles++; + } + } + SetDiagnosticElapsed ("deploy.fastdeploy2.compare.ms", phase); + SetDiagnosticProperty ("deploy.fastdeploy3.missing.files", missingFiles); + + if (!await RemoveStaleOverrideFiles (overridePath, expectedFiles, overrideFiles)) { + return false; + } + + return await CopyChangedFiles (remoteStagingPath, overridePath, changedFiles); + } + + List PrepareDirectPushFiles () + { + var files = new List (); + foreach (var file in FastDevFiles ?? Array.Empty ()) { + if (!File.Exists (file.ItemSpec)) { + LogDebugMessage ($"File '{file.ItemSpec}' does not exists. Skipping."); + continue; + } + if (Path.GetExtension (file.ItemSpec) == ".so") { + string abi = AndroidRidAbiHelper.GetNativeLibraryAbi (file); + if (abi != PrimaryCpuAbi) { + LogDebugMessage ($"NotifySync SkipCopyFile {file.ItemSpec} abi not suitable for this device."); + continue; + } + } + + files.Add (new DirectPushFile { + LocalPath = file.ItemSpec, + RelativePath = GetAdbPushTargetPath (file), + }); + LogDiagnostic ($"Prepared {file.ItemSpec} => {files [files.Count - 1].RelativePath}"); + } + + if (EnvironmentFiles?.Length > 0) { + byte [] environmentData = CreateEnvironmentFileData (EnvironmentFiles, out DateTime newestFileDateTime); + if (environmentData.Length > 0) { + string environmentFile = Path.Combine (GetFullPath (IntermediateOutputPath), "fastdeploy4", PrimaryCpuAbi, "environment"); + Directory.CreateDirectory (Path.GetDirectoryName (environmentFile)); + File.WriteAllBytes (environmentFile, environmentData); + File.SetLastWriteTimeUtc (environmentFile, newestFileDateTime); + files.Add (new DirectPushFile { + LocalPath = environmentFile, + RelativePath = $"{PrimaryCpuAbi}/environment", + }); + } + } + + return files; + } + + async Task CreateRemoteStagingDirectories (string remoteStagingPath, List files) + { + var directories = new HashSet (StringComparer.Ordinal) { remoteStagingPath }; + foreach (var file in files) { + string directory = Path.GetDirectoryName (file.RelativePath)?.Replace ("\\", "/") ?? ""; + if (!string.IsNullOrEmpty (directory)) { + directories.Add ($"{remoteStagingPath}/{directory}"); + } + } + + foreach (var batch in BatchArguments ("mkdir", "-p", directories)) { + var result = await RunAdbCommand (new [] { "shell" }.Concat (batch).ToArray ()); + if (result.ExitCode != 0 || IsShellError (result.Output, "mkdir")) { + LogFastDeploy2Error ("XA0129", result.Output, remoteStagingPath); + return false; + } + } + return true; + } + + async Task> PushFilesOneByOne (string remoteStagingPath, List files) + { + var changedFiles = new HashSet (StringComparer.Ordinal); + int pushed = 0; + int skipped = 0; + foreach (var file in files) { + var args = CreatePushArgs (file.LocalPath, $"{remoteStagingPath}/{file.RelativePath}"); + var result = await RunAdbCommand (args.ToArray ()); + if (result.ExitCode != 0) { + LogFastDeploy2Error ("XA0129", result.Output, file.LocalPath); + return null; + } + var counts = TryParsePushSummary (result.Output); + pushed += counts.pushed; + skipped += counts.skipped; + if (counts.pushed > 0) { + changedFiles.Add (file.RelativePath); + } + LogDiagnostic (result.Output); + } + SetDiagnosticProperty ("deploy.fastdeploy2.adb.pushed.files", pushed); + SetDiagnosticProperty ("deploy.fastdeploy2.adb.skipped.files", skipped); + SetDiagnosticProperty ("deploy.fastdeploy4.direct.push.files", files.Count); + return changedFiles; + } + + async Task PushFilesInBulk (string remoteStagingPath, List files) + { + int pushed = 0; + int skipped = 0; + int batches = 0; + foreach (var group in files.GroupBy (file => Path.GetDirectoryName (file.RelativePath)?.Replace ("\\", "/") ?? "", StringComparer.Ordinal)) { + string remoteDirectory = string.IsNullOrEmpty (group.Key) ? remoteStagingPath : $"{remoteStagingPath}/{group.Key}"; + foreach (var batch in BatchPushFiles (group.ToList (), remoteDirectory)) { + var result = await RunAdbCommand (batch.ToArray ()); + if (result.ExitCode != 0) { + LogFastDeploy2Error ("XA0129", result.Output, remoteDirectory); + return false; + } + var counts = TryParsePushSummary (result.Output); + pushed += counts.pushed; + skipped += counts.skipped; + batches++; + LogDiagnostic (result.Output); + } + } + SetDiagnosticProperty ("deploy.fastdeploy2.adb.pushed.files", pushed); + SetDiagnosticProperty ("deploy.fastdeploy2.adb.skipped.files", skipped); + SetDiagnosticProperty ("deploy.fastdeploy4.bulk.batches", batches); + return true; + } + + HashSet GetChangedFiles (Dictionary stagedFiles, Dictionary overrideFiles) + { + var changedFiles = new HashSet (StringComparer.Ordinal); + foreach (var file in stagedFiles) { + if (!overrideFiles.TryGetValue (file.Key, out RemoteFileInfo existing) || + existing.Size != file.Value.Size || + existing.ModifiedTime != file.Value.ModifiedTime) { + changedFiles.Add (file.Key); + } + } + return changedFiles; + } + + async Task> GetOverrideFileList (string overridePath) + { + string output = await RunAs ("find", overridePath, "-type", "f"); + if (RaiseRunAsError (output)) { + return null; + } + if (IsMissingDirectoryError (output)) { + return new HashSet (StringComparer.Ordinal); + } + if (IsShellError (output, "find")) { + LogFastDeploy2Error ("XA0129", output, overridePath); + return null; + } + + var files = new HashSet (StringComparer.Ordinal); + string prefix = overridePath.TrimEnd ('/') + "/"; + foreach (string line in output.Split (new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) { + string remoteFile = line.Trim (); + if (remoteFile.StartsWith (prefix, StringComparison.Ordinal)) { + files.Add (remoteFile.Substring (prefix.Length)); + } + } + return files; + } + + async Task RemoveStaleOverrideFiles (string overridePath, IEnumerable stagedFiles, IEnumerable overrideFiles) + { + var phase = Stopwatch.StartNew (); + var staged = new HashSet (stagedFiles, StringComparer.Ordinal); + var staleFiles = new List (); + foreach (var file in overrideFiles) { + if (!staged.Contains (file)) { + staleFiles.Add ($"{overridePath}/{file}"); + } + } + + SetDiagnosticProperty ("deploy.fastdeploy2.stale.files", staleFiles.Count); + foreach (var batch in BatchArguments ("rm", "-f", staleFiles)) { + string output = await RunAs (batch.ToArray ()); + if (RaiseRunAsError (output) || IsShellError (output, "rm")) { + LogFastDeploy2Error ("XA0129", output, overridePath); + return false; + } + } + SetDiagnosticElapsed ("deploy.fastdeploy2.stale.remove.ms", phase); + return true; + } + + async Task CopyChangedFiles (string remoteStagingPath, string overridePath, HashSet changedFiles) + { + SetDiagnosticProperty ("deploy.fastdeploy2.changed.files", changedFiles.Count); + var filesByDirectory = new Dictionary> (StringComparer.Ordinal); + foreach (string file in changedFiles) { + string directory = Path.GetDirectoryName (file)?.Replace ("\\", "/") ?? ""; + if (!filesByDirectory.TryGetValue (directory, out List files)) { + files = new List (); + filesByDirectory.Add (directory, files); + } + files.Add (file); + } + + var phase = Stopwatch.StartNew (); + foreach (var group in filesByDirectory) { + string targetDirectory = string.IsNullOrEmpty (group.Key) ? overridePath : $"{overridePath}/{group.Key}"; + phase.Restart (); + string output = await RunAs ("mkdir", "-p", targetDirectory); + AddDiagnosticElapsed ("deploy.fastdeploy2.override.mkdir.ms", phase); + if (RaiseRunAsError (output) || IsShellError (output, "mkdir")) { + LogFastDeploy2Error ("XA0129", output, targetDirectory); + return false; + } + + for (int i = 0; i < group.Value.Count; i += CopyBatchSize) { + var args = new List { "cp", "-p" }; + foreach (string file in group.Value.Skip (i).Take (CopyBatchSize)) { + args.Add ($"{remoteStagingPath}/{file}"); + } + args.Add (targetDirectory); + phase.Restart (); + output = await RunAs (args.ToArray ()); + AddDiagnosticElapsed ("deploy.fastdeploy2.override.copy.ms", phase); + if (RaiseRunAsError (output) || IsShellError (output, "cp")) { + LogFastDeploy2Error ("XA0129", output, targetDirectory); + return false; + } + } + } + return true; + } + + List CreatePushArgs (string localPath, string remotePath) + { + var args = new List { "push" }; + if (!string.IsNullOrEmpty (AdbPushCompressionAlgorithm)) { + args.Add ("-z"); + args.Add (AdbPushCompressionAlgorithm); + } + args.Add ("--sync"); + args.Add (localPath); + args.Add (remotePath); + return args; + } + + IEnumerable> BatchPushFiles (List files, string remoteDirectory) + { + var batch = CreatePushArgsPrefix (); + int length = EstimateCommandLength (batch) + remoteDirectory.Length + 4; + foreach (var file in files) { + if (Path.GetFileName (file.LocalPath) != Path.GetFileName (file.RelativePath)) { + var single = CreatePushArgs (file.LocalPath, $"{remoteDirectory}/{Path.GetFileName (file.RelativePath)}"); + yield return single; + continue; + } + + int itemLength = file.LocalPath.Length + 3; + if (batch.Count > 3 && length + itemLength >= MaxAdbCommandLength) { + batch.Add (remoteDirectory); + yield return batch; + batch = CreatePushArgsPrefix (); + length = EstimateCommandLength (batch) + remoteDirectory.Length + 4; + } + batch.Add (file.LocalPath); + length += itemLength; + } + if (batch.Count > 3) { + batch.Add (remoteDirectory); + yield return batch; + } + } + + List CreatePushArgsPrefix () + { + var args = new List { "push" }; + if (!string.IsNullOrEmpty (AdbPushCompressionAlgorithm)) { + args.Add ("-z"); + args.Add (AdbPushCompressionAlgorithm); + } + args.Add ("--sync"); + return args; + } + + IEnumerable> BatchArguments (string command, string option, IEnumerable values) + { + var batch = new List { command, option }; + int length = command.Length + option.Length + 2; + foreach (var value in values) { + int itemLength = value.Length + 3; + if (batch.Count > 2 && length + itemLength >= MaxAdbCommandLength) { + yield return batch; + batch = new List { command, option }; + length = command.Length + option.Length + 2; + } + batch.Add (value); + length += itemLength; + } + if (batch.Count > 2) { + yield return batch; + } + } + + int EstimateCommandLength (List args) + { + int length = 0; + foreach (var arg in args) { + length += arg.Length + 3; + } + return length; + } + + (int pushed, int skipped) TryParsePushSummary (string output) + { + int pushed = 0; + int skipped = 0; + foreach (var line in output.Split (new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) { + var match = System.Text.RegularExpressions.Regex.Match (line, @"(?\d+) files? pushed, (?\d+) skipped"); + if (!match.Success) { + continue; + } + pushed = int.Parse (match.Groups ["pushed"].Value); + skipped = int.Parse (match.Groups ["skipped"].Value); + } + return (pushed, skipped); + } + + string GetAdbPushTargetPath (Microsoft.Build.Framework.ITaskItem file) + { + string targetPath = file.GetMetadata ("TargetPath"); + if (string.IsNullOrEmpty (targetPath)) { + LogDiagnostic ($"'TargetPath' meta data not found on '{file.ItemSpec}'. Falling back to'DestinationSubPath'"); + targetPath = file.GetMetadata ("DestinationSubPath"); + } + if (!string.IsNullOrEmpty (targetPath)) { + return targetPath.Replace ("\\", "/"); + } + return Path.GetFileName (file.ItemSpec); + } + + byte [] CreateEnvironmentFileData (Microsoft.Build.Framework.ITaskItem [] environments, out DateTime newestFileDateTime) + { + int maxKeyLength = 0; + int maxValueLength = 0; + newestFileDateTime = DateTime.MinValue; + var data = new Dictionary (); + foreach (var env in environments ?? Array.Empty ()) { + if (!File.Exists (env.ItemSpec)) + continue; + DateTime modifiedDateTime = File.GetLastWriteTimeUtc (env.ItemSpec); + if (modifiedDateTime > newestFileDateTime) + newestFileDateTime = modifiedDateTime; + foreach (string line in File.ReadLines (env.ItemSpec)) { + if (string.IsNullOrEmpty (line)) + continue; + int index = line.IndexOf ('='); + if (index == -1) { + LogDebugMessage ($"Skipping invalid environment line: {line}"); + continue; + } + var key = line.Substring (0, index); + var value = line.Substring (index + 1); + maxKeyLength = Math.Max (maxKeyLength, key.Length); + maxValueLength = Math.Max (maxValueLength, value.Length); + data [key] = value; + } + } + + if (newestFileDateTime == DateTime.MinValue) { + return Array.Empty (); + } + + maxKeyLength++; + maxValueLength++; + + using (var stream = new MemoryStream ()) + using (var binaryWriter = new BinaryWriter (stream, Encoding.ASCII)) { + binaryWriter.Write (Encoding.ASCII.GetBytes ("0x" + maxKeyLength.ToString ("X8") + '\0')); + binaryWriter.Write (Encoding.ASCII.GetBytes ("0x" + maxValueLength.ToString ("X8") + '\0')); + foreach (var kvp in data) { + binaryWriter.Write (Encoding.ASCII.GetBytes (kvp.Key.PadRight (maxKeyLength, '\0'))); + binaryWriter.Write (Encoding.ASCII.GetBytes (kvp.Value.PadRight (maxValueLength, '\0'))); + } + binaryWriter.Flush (); + return stream.ToArray (); + } + } + + class DirectPushFile + { + public string LocalPath { get; set; } + public string RelativePath { get; set; } + } + } +} diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets index d5f457ded45..579436747f2 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets @@ -25,6 +25,7 @@ Copyright (C) 2016 Xamarin. All rights reserved. + @@ -324,11 +325,15 @@ Copyright (C) 2016 Xamarin. All rights reserved. <_AndroidIsTestOnlyPackage Condition=" '$(_AndroidIsTestOnlyPackage)' == '' ">False <_FastDeploymentDiagnosticLogging Condition=" '$(_FastDeploymentDiagnosticLogging)' == '' ">False <_AndroidFastDevStrategy Condition=" '$(_AndroidFastDevStrategy)' == '' ">FastDeploy + <_AndroidFastDeploy4PushMode Condition=" '$(_AndroidFastDeploy4PushMode)' == '' ">SingleFile any + Condition=" '$(_AndroidFastDevStrategy)' != 'FastDeploy' And '$(_AndroidFastDevStrategy)' != 'FastDeploy2' And '$(_AndroidFastDevStrategy)' != 'FastDeploy3' And '$(_AndroidFastDevStrategy)' != 'FastDeploy4' " + Text="Invalid _AndroidFastDevStrategy value '$(_AndroidFastDevStrategy)'. Supported values are 'FastDeploy', 'FastDeploy2', 'FastDeploy3', and 'FastDeploy4'." /> + @@ -406,6 +411,31 @@ Copyright (C) 2016 Xamarin. All rights reserved. IntermediateOutputPath="$(IntermediateOutputPath)" EnvironmentFiles="@(_EnvironmentFiles)" /> + Date: Thu, 18 Jun 2026 09:41:05 +0200 Subject: [PATCH 05/12] Combine FastDeploy package and pid probes Have experimental FastDeploy strategies retrieve the app private path and process id in a single run-as shell invocation so process termination can avoid a separate adb ps call when possible. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tasks/FastDeploy2.cs | 55 ++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs index db8214fe9b2..defe498729c 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs @@ -91,6 +91,7 @@ public string InternalPath { public bool AdbIsRoot { get; set; } = false; public string UserId { get; set; } = null; public string PackageName { get; set; } = null; + public int ProcessId { get; set; } = 0; } class DiagnosticData { @@ -144,6 +145,7 @@ class DiagnosticData { { "deploy.orchestration.run-as-disabled-check.ms", "" }, { "deploy.orchestration.package-check.ensure-user.ms", "" }, { "deploy.orchestration.package-check.run-as-pwd.ms", "" }, + { "deploy.orchestration.package-check.run-as-pwd-pidof.ms", "" }, { "deploy.orchestration.package-check.readlink.ms", "" }, { "deploy.orchestration.package-check.system-app.ms", "" }, { "deploy.orchestration.package-check.evaluate.ms", "" }, @@ -349,10 +351,19 @@ async Task CheckAppInstalledAndDebuggable (string packageName) var phase = Stopwatch.StartNew (); packageInfo.UserId = UserID; packageInfo.PackageName = packageName; + packageInfo.ProcessId = 0; await EnsureUserIsRunning (); SetDiagnosticElapsed ("deploy.orchestration.package-check.ensure-user.ms", phase); phase.Restart (); - packageInfo.InternalPath = packageInfo.InternalPath ?? await RunAs ("pwd"); + string packageInfoOutput = IsSafePackageNameForShell (packageName) ? + await RunAs ("sh", "-c", $"pwd; pidof {packageName} 2>/dev/null || true") : + await RunAs ("pwd"); + SetDiagnosticElapsed ("deploy.orchestration.package-check.run-as-pwd-pidof.ms", phase); + ParsePackageInfoOutput (packageInfoOutput); + if (string.IsNullOrEmpty (packageInfo.InternalPath)) { + packageInfo.InternalPath = packageInfoOutput?.Trim (); + } + phase.Restart (); SetDiagnosticElapsed ("deploy.orchestration.package-check.run-as-pwd.ms", phase); if (packageInfo.InternalPath.IndexOf ("Permission denied", StringComparison.OrdinalIgnoreCase) >= 0) { phase.Restart (); @@ -389,6 +400,43 @@ async Task CheckAppInstalledAndDebuggable (string packageName) SetDiagnosticElapsed ("deploy.orchestration.package-check.evaluate.ms", phase); } + static bool IsSafePackageNameForShell (string packageName) + { + if (string.IsNullOrEmpty (packageName)) { + return false; + } + foreach (char c in packageName) { + if (!(char.IsLetterOrDigit (c) || c == '.' || c == '_')) { + return false; + } + } + return true; + } + + void ParsePackageInfoOutput (string output) + { + if (string.IsNullOrEmpty (output)) { + return; + } + + string [] lines = output.Replace ("\r", "").Split (new char [] { '\n' }, StringSplitOptions.None); + if (lines.Length > 0 && !string.IsNullOrEmpty (lines [0])) { + packageInfo.InternalPath = lines [0].Trim (); + } + if (lines.Length <= 1) { + return; + } + + string pidLine = lines [1].Trim (); + int space = pidLine.IndexOf (' '); + if (space >= 0) { + pidLine = pidLine.Substring (0, space); + } + if (int.TryParse (pidLine, out int pid)) { + packageInfo.ProcessId = pid; + } + } + async Task EnsureUserIsRunning () { var userId = (UserID ?? "").Trim (); @@ -471,7 +519,10 @@ async Task RemoveOverrideDirectory () async Task TerminateApp () { var phase = Stopwatch.StartNew (); - var pid = await Device.GetProcessId (PackageName, CancellationToken); + var pid = packageInfo.ProcessId; + if (pid == 0 && packageInfo.IsSystemApplication) { + pid = await Device.GetProcessId (PackageName, CancellationToken); + } SetDiagnosticElapsed ("deploy.orchestration.terminate.get-pid.ms", phase); if (pid == 0) { LogDebugMessage ($"{PackageName} was not running, skipping kill"); From 7a7b9e0d07a3f938238ed7a10a01c7f4064c81b0 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Thu, 18 Jun 2026 09:54:08 +0200 Subject: [PATCH 06/12] Add experimental symlink app transfer mode Allow FastDeploy2-derived strategies to replace app-private copies with symlinks to staged files so the app storage transfer cost can be measured separately from upload and staging. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tasks/FastDeploy2.cs | 156 ++++++++++++++++++ .../Xamarin.Android.Common.Debugging.targets | 7 + 2 files changed, 163 insertions(+) diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs index defe498729c..4a9950b7896 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs @@ -23,6 +23,7 @@ public class FastDeploy2 : AsyncTask const string OverridePath = "files/.__override__"; const int StaleFileRemovalBatchSize = 100; const int CopyBatchSize = 25; + const int MaxShellCommandLength = 900; public override string TaskPrefix => "FD2"; @@ -68,6 +69,8 @@ public class FastDeploy2 : AsyncTask public string AdbPushCompressionAlgorithm { get; set; } = "any"; + public string AppFileTransferMode { get; set; } = "Copy"; + AndroidDevice Device; PackageInfo packageInfo = new PackageInfo (); DateTime lastUpload = DateTime.MinValue; @@ -156,6 +159,9 @@ class DiagnosticData { { "deploy.orchestration.install.retry-reinstall.ms", "" }, { "deploy.orchestration.terminate.get-pid.ms", "" }, { "deploy.orchestration.terminate.kill.ms", "" }, + { "deploy.app.file.transfer.mode", "" }, + { "deploy.symlink.created.files", "" }, + { "deploy.symlink.removed.files", "" }, { "pii.deploy.error", "" }, { "pii.deploy.file", "" }, }; @@ -330,6 +336,7 @@ async Task RunInstall () } SetDiagnosticElapsed ("deploy.orchestration.empty-check.ms", phase); + diagnosticData.SetProperty ("deploy.app.file.transfer.mode", AppFileTransferMode); phase.Restart (); await TerminateApp (); SetDiagnosticElapsed ("deploy.orchestration.terminate.ms", phase); @@ -565,6 +572,10 @@ protected virtual async Task DeployFastDevFilesWithAdbPush (string overrid } SetDiagnosticElapsed ("deploy.fastdeploy2.upload.ms", phase); + if (UseSymlinkAppFileTransfer ()) { + return await UpdateOverrideSymlinks (remoteStagingPath, overridePath, stagedFiles); + } + phase.Restart (); var stagedFileData = await GetRemoteFileData (remoteStagingPath, runAs: false); SetDiagnosticElapsed ("deploy.fastdeploy2.staging.stat.ms", phase); @@ -586,6 +597,11 @@ protected virtual async Task DeployFastDevFilesWithAdbPush (string overrid return await CopyChangedFiles (remoteStagingPath, overridePath, stagedFileData, overrideFileData); } + protected virtual bool UseSymlinkAppFileTransfer () + { + return string.Equals (AppFileTransferMode, "Symlink", StringComparison.OrdinalIgnoreCase); + } + protected HashSet PrepareAdbPushStagingDirectory (string stagingDirectory) { if (Directory.Exists (stagingDirectory)) { @@ -817,6 +833,127 @@ async Task RemoveStaleOverrideFiles (string overridePath, Dictionary UpdateOverrideSymlinks (string remoteStagingPath, string overridePath, HashSet stagedFiles) + { + var phase = Stopwatch.StartNew (); + var overrideEntries = await GetOverrideEntries (overridePath, includeSymlinks: false); + if (overrideEntries == null) { + return false; + } + SetDiagnosticElapsed ("deploy.fastdeploy2.override.stat.ms", phase); + + phase.Restart (); + var overrideSymlinks = await GetOverrideEntries (overridePath, includeSymlinks: true); + if (overrideSymlinks == null) { + return false; + } + AddDiagnosticElapsed ("deploy.fastdeploy2.override.stat.ms", phase); + + phase.Restart (); + var staleFiles = new List (); + foreach (var entry in overrideEntries) { + if (!stagedFiles.Contains (entry)) { + staleFiles.Add ($"{overridePath}/{entry}"); + } + } + + var missingSymlinks = new List (); + foreach (var entry in stagedFiles) { + if (!overrideSymlinks.Contains (entry)) { + missingSymlinks.Add (entry); + } + } + SetDiagnosticElapsed ("deploy.fastdeploy2.compare.ms", phase); + SetDiagnosticProperty ("deploy.fastdeploy2.changed.files", missingSymlinks.Count); + SetDiagnosticProperty ("deploy.symlink.created.files", missingSymlinks.Count); + SetDiagnosticProperty ("deploy.symlink.removed.files", staleFiles.Count + missingSymlinks.Count); + SetDiagnosticProperty ("deploy.fastdeploy2.stale.files", staleFiles.Count); + + phase.Restart (); + if (!await RemoveOverridePaths (staleFiles.Concat (missingSymlinks.Select (entry => $"{overridePath}/{entry}")))) { + return false; + } + SetDiagnosticElapsed ("deploy.fastdeploy2.stale.remove.ms", phase); + + return await CreateOverrideSymlinks (remoteStagingPath, overridePath, missingSymlinks); + } + + async Task> GetOverrideEntries (string overridePath, bool includeSymlinks) + { + string findExpression = includeSymlinks ? + $"find {overridePath} -type l -print" : + $"find {overridePath} \\( -type f -o -type l \\) -print"; + string output = await RunAs ("sh", "-c", findExpression); + if (RaiseRunAsError (output)) { + return null; + } + if (IsMissingDirectoryError (output)) { + return new HashSet (StringComparer.Ordinal); + } + if (IsShellError (output, "find")) { + LogFastDeploy2Error ("XA0129", output, overridePath); + return null; + } + + var entries = new HashSet (StringComparer.Ordinal); + string prefix = overridePath.TrimEnd ('/') + "/"; + foreach (string line in output.Split (new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) { + string remoteFile = line.Trim (); + if (remoteFile.StartsWith (prefix, StringComparison.Ordinal)) { + entries.Add (remoteFile.Substring (prefix.Length)); + } + } + return entries; + } + + async Task RemoveOverridePaths (IEnumerable paths) + { + foreach (var batch in BatchArguments ("rm", "-f", paths)) { + string output = await RunAs (batch.ToArray ()); + if (RaiseRunAsError (output) || IsShellError (output, "rm")) { + LogFastDeploy2Error ("XA0129", output, string.Join (" ", batch)); + return false; + } + } + return true; + } + + async Task CreateOverrideSymlinks (string remoteStagingPath, string overridePath, List missingSymlinks) + { + var filesByDirectory = new Dictionary> (StringComparer.Ordinal); + foreach (string file in missingSymlinks) { + string directory = Path.GetDirectoryName (file)?.Replace ("\\", "/") ?? ""; + if (!filesByDirectory.TryGetValue (directory, out List files)) { + files = new List (); + filesByDirectory.Add (directory, files); + } + files.Add (file); + } + + var phase = Stopwatch.StartNew (); + foreach (var group in filesByDirectory) { + string targetDirectory = string.IsNullOrEmpty (group.Key) ? overridePath : $"{overridePath}/{group.Key}"; + string sourcePattern = string.IsNullOrEmpty (group.Key) ? $"{remoteStagingPath}/*" : $"{remoteStagingPath}/{group.Key}/*"; + phase.Restart (); + string output = await RunAs ("mkdir", "-p", targetDirectory); + AddDiagnosticElapsed ("deploy.fastdeploy2.override.mkdir.ms", phase); + if (RaiseRunAsError (output) || IsShellError (output, "mkdir")) { + LogFastDeploy2Error ("XA0129", output, targetDirectory); + return false; + } + + phase.Restart (); + output = await RunAs ("sh", "-c", $"ln -sf {sourcePattern} {targetDirectory}/"); + AddDiagnosticElapsed ("deploy.fastdeploy2.override.copy.ms", phase); + if (RaiseRunAsError (output) || IsShellError (output, "ln")) { + LogFastDeploy2Error ("XA0129", output, targetDirectory); + return false; + } + } + + return true; + } + async Task CopyChangedFiles (string remoteStagingPath, string overridePath, Dictionary stagedFiles, Dictionary overrideFiles) { var phase = Stopwatch.StartNew (); @@ -871,6 +1008,25 @@ async Task CopyChangedFiles (string remoteStagingPath, string overridePath return true; } + IEnumerable> BatchArguments (string command, string option, IEnumerable values) + { + var batch = new List { command, option }; + int length = command.Length + option.Length + 2; + foreach (var value in values) { + int itemLength = value.Length + 3; + if (batch.Count > 2 && length + itemLength >= MaxShellCommandLength) { + yield return batch; + batch = new List { command, option }; + length = command.Length + option.Length + 2; + } + batch.Add (value); + length += itemLength; + } + if (batch.Count > 2) { + yield return batch; + } + } + protected void SetDiagnosticElapsed (string key, Stopwatch stopwatch) { diagnosticData.SetProperty (key, stopwatch.ElapsedMilliseconds); diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets index 579436747f2..52221120045 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets @@ -326,6 +326,7 @@ Copyright (C) 2016 Xamarin. All rights reserved. <_FastDeploymentDiagnosticLogging Condition=" '$(_FastDeploymentDiagnosticLogging)' == '' ">False <_AndroidFastDevStrategy Condition=" '$(_AndroidFastDevStrategy)' == '' ">FastDeploy <_AndroidFastDeploy4PushMode Condition=" '$(_AndroidFastDeploy4PushMode)' == '' ">SingleFile + <_AndroidFastDeployAppFileTransferMode Condition=" '$(_AndroidFastDeployAppFileTransferMode)' == '' ">Copy any + @@ -380,6 +384,7 @@ Copyright (C) 2016 Xamarin. All rights reserved. AdbToolPath="$(AdbToolPath)" AdbToolExe="$(AdbToolExe)" AdbPushCompressionAlgorithm="$(AndroidFastDeploymentAdbCompressionAlgorithm)" + AppFileTransferMode="$(_AndroidFastDeployAppFileTransferMode)" UploadFlagFile="$(_UploadFlag)" UsingAndroidNETSdk="$(UsingAndroidNETSdk)" UserID="$(AndroidDeviceUserId)" @@ -404,6 +409,7 @@ Copyright (C) 2016 Xamarin. All rights reserved. AdbToolPath="$(AdbToolPath)" AdbToolExe="$(AdbToolExe)" AdbPushCompressionAlgorithm="$(AndroidFastDeploymentAdbCompressionAlgorithm)" + AppFileTransferMode="$(_AndroidFastDeployAppFileTransferMode)" UploadFlagFile="$(_UploadFlag)" UsingAndroidNETSdk="$(UsingAndroidNETSdk)" UserID="$(AndroidDeviceUserId)" @@ -429,6 +435,7 @@ Copyright (C) 2016 Xamarin. All rights reserved. AdbToolExe="$(AdbToolExe)" AdbPushCompressionAlgorithm="$(AndroidFastDeploymentAdbCompressionAlgorithm)" PushMode="$(_AndroidFastDeploy4PushMode)" + AppFileTransferMode="$(_AndroidFastDeployAppFileTransferMode)" UploadFlagFile="$(_UploadFlag)" UsingAndroidNETSdk="$(UsingAndroidNETSdk)" UserID="$(AndroidDeviceUserId)" From c630e7d8643c1da33d7b489893b82ef3355285dd Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Thu, 18 Jun 2026 10:05:48 +0200 Subject: [PATCH 07/12] Make FastDeploy2 use direct push symlinks Convert the experimental FastDeploy2 strategy to bulk-push original FastDev inputs directly to /tmp/fastdev2 without host-side staging, and use symlinks in files/.__override__ by default. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tasks/FastDeploy2.cs | 169 +++++++++++++++++- .../Tasks/FastDeploy4.cs | 4 +- .../Xamarin.Android.Common.Debugging.targets | 1 + 3 files changed, 166 insertions(+), 8 deletions(-) diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs index 4a9950b7896..8b8d48158fe 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs @@ -192,6 +192,11 @@ protected class RemoteFileInfo { public long ModifiedTime { get; set; } } + protected class DirectPushFile { + public string LocalPath { get; set; } + public string RelativePath { get; set; } + } + void DebugHandler (string task, string message) { LogDiagnostic ($"DEBUG {task} {message}"); @@ -544,20 +549,20 @@ async Task TerminateApp () protected virtual async Task DeployFastDevFilesWithAdbPush (string overridePath) { - string stagingDirectory = GetLocalStagingDirectory (); var phase = Stopwatch.StartNew (); - var stagedFiles = PrepareAdbPushStagingDirectory (stagingDirectory); + var directPushFiles = PrepareDirectPushFiles (); + var stagedFiles = new HashSet (directPushFiles.Select (file => file.RelativePath), StringComparer.Ordinal); SetDiagnosticElapsed ("deploy.fastdeploy2.local.stage.ms", phase); if (stagedFiles.Count == 0) { - LogDiagnostic ("No FastDev files were staged for adb push deployment."); + LogDiagnostic ("No FastDev files were prepared for adb push deployment."); return true; } string remoteStagingPath = GetRemoteAdbPushStagingPath (); phase.Restart (); - string output = await Device.RunShellCommand (CancellationToken, "mkdir", "-p", remoteStagingPath); + string output = await CreateRemoteStagingDirectories (remoteStagingPath, stagedFiles); SetDiagnosticElapsed ("deploy.fastdeploy2.remote.mkdir.ms", phase); - if (IsShellError (output, "mkdir")) { + if (!string.IsNullOrEmpty (output) && IsShellError (output, "mkdir")) { LogFastDeploy2Error ("XA0129", output, remoteStagingPath); return false; } @@ -567,7 +572,7 @@ protected virtual async Task DeployFastDevFilesWithAdbPush (string overrid } phase.Restart (); - if (!await UploadStagingDirectory (stagingDirectory, remoteStagingPath)) { + if (!await UploadFiles (remoteStagingPath, directPushFiles)) { return false; } SetDiagnosticElapsed ("deploy.fastdeploy2.upload.ms", phase); @@ -597,6 +602,63 @@ protected virtual async Task DeployFastDevFilesWithAdbPush (string overrid return await CopyChangedFiles (remoteStagingPath, overridePath, stagedFileData, overrideFileData); } + async Task CreateRemoteStagingDirectories (string remoteStagingPath, HashSet stagedFiles) + { + var directories = new HashSet (StringComparer.Ordinal) { remoteStagingPath }; + foreach (var file in stagedFiles) { + string directory = Path.GetDirectoryName (file)?.Replace ("\\", "/") ?? ""; + if (!string.IsNullOrEmpty (directory)) { + directories.Add ($"{remoteStagingPath}/{directory}"); + } + } + + var output = new StringBuilder (); + foreach (var batch in BatchArguments ("mkdir", "-p", directories)) { + output.Append (await Device.RunShellCommand (CancellationToken, batch.ToArray ())); + } + return output.ToString (); + } + + protected List PrepareDirectPushFiles () + { + var files = new List (); + foreach (var file in FastDevFiles ?? Array.Empty ()) { + if (!File.Exists (file.ItemSpec)) { + LogDebugMessage ($"File '{file.ItemSpec}' does not exists. Skipping."); + continue; + } + if (Path.GetExtension (file.ItemSpec) == ".so") { + string abi = AndroidRidAbiHelper.GetNativeLibraryAbi (file); + if (abi != PrimaryCpuAbi) { + LogDebugMessage ($"NotifySync SkipCopyFile {file.ItemSpec} abi not suitable for this device."); + continue; + } + } + + files.Add (new DirectPushFile { + LocalPath = GetFullPath (file.ItemSpec), + RelativePath = GetAdbPushTargetPath (file), + }); + LogDiagnostic ($"Prepared {file.ItemSpec} => {files [files.Count - 1].RelativePath}"); + } + + if (EnvironmentFiles?.Length > 0) { + byte [] environmentData = CreateEnvironmentFileData (EnvironmentFiles, out DateTime newestFileDateTime); + if (environmentData.Length > 0) { + string environmentFile = Path.Combine (GetFullPath (IntermediateOutputPath), "fastdeploy2-environment", PrimaryCpuAbi, "environment"); + Directory.CreateDirectory (Path.GetDirectoryName (environmentFile)); + File.WriteAllBytes (environmentFile, environmentData); + File.SetLastWriteTimeUtc (environmentFile, newestFileDateTime); + files.Add (new DirectPushFile { + LocalPath = environmentFile, + RelativePath = $"{PrimaryCpuAbi}/environment", + }); + } + } + + return files; + } + protected virtual bool UseSymlinkAppFileTransfer () { return string.Equals (AppFileTransferMode, "Symlink", StringComparison.OrdinalIgnoreCase); @@ -1076,6 +1138,101 @@ protected virtual async Task UploadStagingDirectory (string stagingDirecto return true; } + protected async Task UploadFiles (string remoteStagingPath, List files) + { + int pushed = 0; + int skipped = 0; + int batches = 0; + foreach (var group in files.GroupBy (file => Path.GetDirectoryName (file.RelativePath)?.Replace ("\\", "/") ?? "", StringComparer.Ordinal)) { + string remoteDirectory = string.IsNullOrEmpty (group.Key) ? remoteStagingPath : $"{remoteStagingPath}/{group.Key}"; + foreach (var batch in BatchPushFiles (group.ToList (), remoteDirectory)) { + var result = await RunAdbCommand (batch.ToArray ()); + if (result.ExitCode != 0) { + LogFastDeploy2Error ("XA0129", result.Output, remoteDirectory); + return false; + } + var counts = TryParsePushSummary (result.Output); + pushed += counts.pushed; + skipped += counts.skipped; + batches++; + LogDiagnostic (result.Output); + } + } + SetDiagnosticProperty ("deploy.fastdeploy2.adb.pushed.files", pushed); + SetDiagnosticProperty ("deploy.fastdeploy2.adb.skipped.files", skipped); + SetDiagnosticProperty ("deploy.fastdeploy4.bulk.batches", batches); + return true; + } + + IEnumerable> BatchPushFiles (List files, string remoteDirectory) + { + var batch = CreatePushArgsPrefix (); + int length = EstimateCommandLength (batch) + remoteDirectory.Length + 4; + foreach (var file in files) { + if (Path.GetFileName (file.LocalPath) != Path.GetFileName (file.RelativePath)) { + yield return CreatePushArgs (file.LocalPath, $"{remoteDirectory}/{Path.GetFileName (file.RelativePath)}"); + continue; + } + + int itemLength = file.LocalPath.Length + 3; + if (batch.Count > 3 && length + itemLength >= 4096) { + batch.Add (remoteDirectory); + yield return batch; + batch = CreatePushArgsPrefix (); + length = EstimateCommandLength (batch) + remoteDirectory.Length + 4; + } + batch.Add (file.LocalPath); + length += itemLength; + } + if (batch.Count > 3) { + batch.Add (remoteDirectory); + yield return batch; + } + } + + List CreatePushArgs (string localPath, string remotePath) + { + var args = CreatePushArgsPrefix (); + args.Add (localPath); + args.Add (remotePath); + return args; + } + + List CreatePushArgsPrefix () + { + var args = new List { "push" }; + if (!string.IsNullOrEmpty (AdbPushCompressionAlgorithm)) { + args.Add ("-z"); + args.Add (AdbPushCompressionAlgorithm); + } + args.Add ("--sync"); + return args; + } + + int EstimateCommandLength (List args) + { + int length = 0; + foreach (var arg in args) { + length += arg.Length + 3; + } + return length; + } + + (int pushed, int skipped) TryParsePushSummary (string output) + { + int pushed = 0; + int skipped = 0; + foreach (var line in output.Split (new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) { + var match = AdbPushSummaryRegex.Match (line); + if (!match.Success) { + continue; + } + pushed = int.Parse (match.Groups ["pushed"].Value); + skipped = int.Parse (match.Groups ["skipped"].Value); + } + return (pushed, skipped); + } + protected void SetAdbPushFileCounts (string output) { var match = AdbPushSummaryRegex.Match (output ?? ""); diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs index e3ce3ca78b3..bc0d1cf4278 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs @@ -110,7 +110,7 @@ protected override async Task DeployFastDevFilesWithAdbPush (string overri return await CopyChangedFiles (remoteStagingPath, overridePath, changedFiles); } - List PrepareDirectPushFiles () + new List PrepareDirectPushFiles () { var files = new List (); foreach (var file in FastDevFiles ?? Array.Empty ()) { @@ -480,7 +480,7 @@ byte [] CreateEnvironmentFileData (Microsoft.Build.Framework.ITaskItem [] enviro } } - class DirectPushFile + new class DirectPushFile { public string LocalPath { get; set; } public string RelativePath { get; set; } diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets index 52221120045..5f64b1f6ffc 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets @@ -326,6 +326,7 @@ Copyright (C) 2016 Xamarin. All rights reserved. <_FastDeploymentDiagnosticLogging Condition=" '$(_FastDeploymentDiagnosticLogging)' == '' ">False <_AndroidFastDevStrategy Condition=" '$(_AndroidFastDevStrategy)' == '' ">FastDeploy <_AndroidFastDeploy4PushMode Condition=" '$(_AndroidFastDeploy4PushMode)' == '' ">SingleFile + <_AndroidFastDeployAppFileTransferMode Condition=" '$(_AndroidFastDeployAppFileTransferMode)' == '' And '$(_AndroidFastDevStrategy)' == 'FastDeploy2' ">Symlink <_AndroidFastDeployAppFileTransferMode Condition=" '$(_AndroidFastDeployAppFileTransferMode)' == '' ">Copy any From dc6f4ddc3f9fbb14c80c7e068cfaa8a9f66b0104 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Thu, 18 Jun 2026 10:19:17 +0200 Subject: [PATCH 08/12] Reduce direct push preparation IO Avoid probing normal FastDev source files before adb push and keep the generated environment blob timestamp-stable by only rewriting it when content changes. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tasks/FastDeploy2.cs | 20 ++++++++++++------- .../Tasks/FastDeploy4.cs | 8 +------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs index 8b8d48158fe..e3da996b93d 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs @@ -623,10 +623,6 @@ protected List PrepareDirectPushFiles () { var files = new List (); foreach (var file in FastDevFiles ?? Array.Empty ()) { - if (!File.Exists (file.ItemSpec)) { - LogDebugMessage ($"File '{file.ItemSpec}' does not exists. Skipping."); - continue; - } if (Path.GetExtension (file.ItemSpec) == ".so") { string abi = AndroidRidAbiHelper.GetNativeLibraryAbi (file); if (abi != PrimaryCpuAbi) { @@ -646,9 +642,7 @@ protected List PrepareDirectPushFiles () byte [] environmentData = CreateEnvironmentFileData (EnvironmentFiles, out DateTime newestFileDateTime); if (environmentData.Length > 0) { string environmentFile = Path.Combine (GetFullPath (IntermediateOutputPath), "fastdeploy2-environment", PrimaryCpuAbi, "environment"); - Directory.CreateDirectory (Path.GetDirectoryName (environmentFile)); - File.WriteAllBytes (environmentFile, environmentData); - File.SetLastWriteTimeUtc (environmentFile, newestFileDateTime); + WriteFileIfChanged (environmentFile, environmentData, newestFileDateTime); files.Add (new DirectPushFile { LocalPath = environmentFile, RelativePath = $"{PrimaryCpuAbi}/environment", @@ -659,6 +653,18 @@ protected List PrepareDirectPushFiles () return files; } + protected bool WriteFileIfChanged (string path, byte [] contents, DateTime modifiedDateTime) + { + if (File.Exists (path) && File.ReadAllBytes (path).SequenceEqual (contents)) { + return false; + } + + Directory.CreateDirectory (Path.GetDirectoryName (path)); + File.WriteAllBytes (path, contents); + File.SetLastWriteTimeUtc (path, modifiedDateTime); + return true; + } + protected virtual bool UseSymlinkAppFileTransfer () { return string.Equals (AppFileTransferMode, "Symlink", StringComparison.OrdinalIgnoreCase); diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs index bc0d1cf4278..9e7d239f227 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs @@ -114,10 +114,6 @@ protected override async Task DeployFastDevFilesWithAdbPush (string overri { var files = new List (); foreach (var file in FastDevFiles ?? Array.Empty ()) { - if (!File.Exists (file.ItemSpec)) { - LogDebugMessage ($"File '{file.ItemSpec}' does not exists. Skipping."); - continue; - } if (Path.GetExtension (file.ItemSpec) == ".so") { string abi = AndroidRidAbiHelper.GetNativeLibraryAbi (file); if (abi != PrimaryCpuAbi) { @@ -137,9 +133,7 @@ protected override async Task DeployFastDevFilesWithAdbPush (string overri byte [] environmentData = CreateEnvironmentFileData (EnvironmentFiles, out DateTime newestFileDateTime); if (environmentData.Length > 0) { string environmentFile = Path.Combine (GetFullPath (IntermediateOutputPath), "fastdeploy4", PrimaryCpuAbi, "environment"); - Directory.CreateDirectory (Path.GetDirectoryName (environmentFile)); - File.WriteAllBytes (environmentFile, environmentData); - File.SetLastWriteTimeUtc (environmentFile, newestFileDateTime); + WriteFileIfChanged (environmentFile, environmentData, newestFileDateTime); files.Add (new DirectPushFile { LocalPath = environmentFile, RelativePath = $"{PrimaryCpuAbi}/environment", From 67d77d45ddad949e89b9f0b245a1ba78eefd6349 Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Thu, 18 Jun 2026 10:31:38 +0200 Subject: [PATCH 09/12] Add maui.link symlink helper Add a native maui.link helper for the experimental symlink transfer path and wire FastDeploy2 to install and invoke it for app override symlink mirroring. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../installers/create-installers.targets | 1 + .../Tasks/FastDeploy2.cs | 107 +++++- .../Tasks/FastDeploy3.cs | 5 - tools/fastdev/fastdevtools.projitems | 6 + tools/fastdev/maui.link/CMakeLists.txt | 34 ++ tools/fastdev/maui.link/main.c | 320 ++++++++++++++++++ 6 files changed, 466 insertions(+), 7 deletions(-) create mode 100644 tools/fastdev/maui.link/CMakeLists.txt create mode 100644 tools/fastdev/maui.link/main.c diff --git a/build-tools/installers/create-installers.targets b/build-tools/installers/create-installers.targets index 9c5e22940f6..bbafa3fc570 100644 --- a/build-tools/installers/create-installers.targets +++ b/build-tools/installers/create-installers.targets @@ -270,6 +270,7 @@ <_MSBuildFiles Include="@(AndroidSupportedTargetJitAbi->'$(MicrosoftAndroidSdkOutDir)lib\%(Identity)\xamarin.find')" /> <_MSBuildFiles Include="@(AndroidSupportedTargetJitAbi->'$(MicrosoftAndroidSdkOutDir)lib\%(Identity)\xamarin.stat')" /> <_MSBuildFiles Include="@(AndroidSupportedTargetJitAbi->'$(MicrosoftAndroidSdkOutDir)lib\%(Identity)\xamarin.cp')" /> + <_MSBuildFiles Include="@(AndroidSupportedTargetJitAbi->'$(MicrosoftAndroidSdkOutDir)lib\%(Identity)\maui.link')" /> InstallLinkTool () + { + if (string.Equals (packageInfo.LinkToolVersion, ToolVersion, StringComparison.OrdinalIgnoreCase)) { + return true; + } + + string output = await RunAs ("cat", $"{ToolsFullPath}/{LinkTool}.version"); + if (string.Equals (output.Trim (), ToolVersion, StringComparison.OrdinalIgnoreCase)) { + packageInfo.LinkToolVersion = ToolVersion; + return true; + } + + string toolAbi = string.IsNullOrEmpty (ToolsAbi) ? PrimaryCpuAbi : ToolsAbi; + string localToolPath = Path.Combine (FastDevToolPath, toolAbi, LinkTool); + string remoteToolDirectory = $"{XAToolsTempPath}/{PackageName}/{GetUserId ()}"; + string remoteToolPath = $"{remoteToolDirectory}/{LinkTool}"; + var result = await RunAdbCommand ("shell", "mkdir", "-p", remoteToolDirectory); + if (result.ExitCode != 0 || IsShellError (result.Output, "mkdir")) { + LogFastDeploy2Error ("XA0129", result.Output, remoteToolDirectory); + return false; + } + + result = await RunAdbCommand ("push", localToolPath, remoteToolPath); + if (result.ExitCode != 0) { + LogFastDeploy2Error ("XA0129", result.Output, localToolPath); + return false; + } + + output = await RunAs ("mkdir", "-p", ToolsFullPath); + if (RaiseRunAsError (output) || IsShellError (output, "mkdir")) { + LogFastDeploy2Error ("XA0129", output, ToolsFullPath); + return false; + } + output = await RunAs ("cp", remoteToolPath, $"{ToolsFullPath}/{LinkTool}"); + if (RaiseRunAsError (output) || IsShellError (output, "cp")) { + LogFastDeploy2Error ("XA0129", output, $"{ToolsFullPath}/{LinkTool}"); + return false; + } + output = await RunAs ("chmod", "700", $"{ToolsFullPath}/{LinkTool}"); + if (RaiseRunAsError (output) || IsShellError (output, "chmod")) { + LogFastDeploy2Error ("XA0129", output, $"{ToolsFullPath}/{LinkTool}"); + return false; + } + output = await RunAs ("sh", "-c", $"printf '%s\\n' '{ToolVersion}' > {ToolsFullPath}/{LinkTool}.version"); + if (RaiseRunAsError (output)) { + return false; + } + packageInfo.LinkToolVersion = ToolVersion; + return true; + } + protected HashSet PrepareAdbPushStagingDirectory (string stagingDirectory) { if (Directory.Exists (stagingDirectory)) { @@ -903,6 +965,26 @@ async Task RemoveStaleOverrideFiles (string overridePath, Dictionary UpdateOverrideSymlinks (string remoteStagingPath, string overridePath, HashSet stagedFiles) { + var toolInstallPhase = Stopwatch.StartNew (); + bool linkToolInstalled = await InstallLinkTool (); + SetDiagnosticElapsed ("deploy.symlink.tool.install.ms", toolInstallPhase); + if (linkToolInstalled) { + var toolPhase = Stopwatch.StartNew (); + string toolOutput = await RunAs ($"{ToolsFullPath}/{LinkTool}", remoteStagingPath, overridePath); + SetDiagnosticElapsed ("deploy.symlink.tool.ms", toolPhase); + SetDiagnosticProperty ("deploy.symlink.tool.result", toolOutput); + if (TryParseLinkToolOutput (toolOutput, out long linked, out long unchanged, out long removed, out long errors) && errors == 0) { + SetDiagnosticProperty ("deploy.symlink.created.files", (int) linked); + SetDiagnosticProperty ("deploy.symlink.removed.files", (int) removed); + SetDiagnosticProperty ("deploy.fastdeploy2.changed.files", (int) linked); + SetDiagnosticProperty ("deploy.fastdeploy2.stale.files", (int) removed); + return true; + } + LogDiagnostic ($"{LinkTool} returned '{toolOutput}'. Falling back to managed symlink update."); + } else { + LogDiagnostic ($"Unable to install {LinkTool}. Falling back to managed symlink update."); + } + var phase = Stopwatch.StartNew (); var overrideEntries = await GetOverrideEntries (overridePath, includeSymlinks: false); if (overrideEntries == null) { @@ -946,6 +1028,22 @@ async Task UpdateOverrideSymlinks (string remoteStagingPath, string overri return await CreateOverrideSymlinks (remoteStagingPath, overridePath, missingSymlinks); } + bool TryParseLinkToolOutput (string output, out long linked, out long unchanged, out long removed, out long errors) + { + linked = 0; + unchanged = 0; + removed = 0; + errors = 0; + var match = LinkToolSummaryRegex.Match (output ?? ""); + if (!match.Success) { + return false; + } + return long.TryParse (match.Groups ["linked"].Value, out linked) && + long.TryParse (match.Groups ["unchanged"].Value, out unchanged) && + long.TryParse (match.Groups ["removed"].Value, out removed) && + long.TryParse (match.Groups ["errors"].Value, out errors); + } + async Task> GetOverrideEntries (string overridePath, bool includeSymlinks) { string findExpression = includeSymlinks ? @@ -1365,8 +1463,12 @@ protected string ResolveAdbPath () protected virtual string GetRemoteAdbPushStagingPath () { - var user = string.IsNullOrEmpty (UserID) ? "0" : UserID; - return $"{RemoteStagingRoot}/{PackageName}/{user}"; + return $"{RemoteStagingRoot}/{PackageName}/{GetUserId ()}"; + } + + protected string GetUserId () + { + return string.IsNullOrEmpty (UserID) ? "0" : UserID; } protected void LogFastDeploy2Error (string errorCode, string error, string file = "") @@ -1718,5 +1820,6 @@ protected struct AdbCommandResult }; static readonly Regex AdbPushSummaryRegex = new Regex (@"(?\d+) files? pushed, (?\d+) skipped", RegexOptions.Compiled); + static readonly Regex LinkToolSummaryRegex = new Regex (@"linked \[(?\d+)\] unchanged \[(?\d+)\] removed \[(?\d+)\] errors \[(?\d+)\]", RegexOptions.Compiled); } } diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy3.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy3.cs index 6e405e31a91..b36dfefc0b7 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy3.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy3.cs @@ -250,10 +250,5 @@ string GetAndroidProductOutDirectory () { return Path.Combine (GetFullPath (IntermediateOutputPath), "fastdeploy3-product-out"); } - - string GetUserId () - { - return string.IsNullOrEmpty (UserID) ? "0" : UserID; - } } } diff --git a/tools/fastdev/fastdevtools.projitems b/tools/fastdev/fastdevtools.projitems index 7ee0d0564ae..e2438580f4f 100644 --- a/tools/fastdev/fastdevtools.projitems +++ b/tools/fastdev/fastdevtools.projitems @@ -43,5 +43,11 @@ $(DebuggingToolsOutputDirectory)\lib\%(FastDevBuildItems.Arch) -DANDROID_NATIVE_API_LEVEL=%(FastDevBuildItems.NativeApiLevel) -DANDROID_PLATFORM=android-%(FastDevBuildItems.NativeApiLevel) + + maui.link + ..\..\..\..\maui.link + $(DebuggingToolsOutputDirectory)\lib\%(FastDevBuildItems.Arch) + -DANDROID_NATIVE_API_LEVEL=%(FastDevBuildItems.NativeApiLevel) -DANDROID_PLATFORM=android-%(FastDevBuildItems.NativeApiLevel) + \ No newline at end of file diff --git a/tools/fastdev/maui.link/CMakeLists.txt b/tools/fastdev/maui.link/CMakeLists.txt new file mode 100644 index 00000000000..f901d2194a4 --- /dev/null +++ b/tools/fastdev/maui.link/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.5) + +project (maui.link C) + +set(CMAKE_C_STANDARD 99) +set(CMAKE_C_STANDARD_REQUIRED ON) + +option(ENABLE_NDK "Build with Android's NDK" OFF) + +if(ENABLE_NDK) + add_definitions("-DPLATFORM_ANDROID") + add_definitions("-DANDROID") + add_definitions("-DLINUX -Dlinux -D__linux__") +endif() + +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) + message("Setting build type to 'Release' as none was specified.") + set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE) + set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" + "MinSizeRel" "Release") +endif() + +set(COMMON_COMPILER_OPTIONS + -O2 + -Wall + -fstack-protector-strong + -fstrict-return + -fno-omit-frame-pointer +) + +add_compile_options("${COMMON_COMPILER_OPTIONS}") + +add_executable(maui.link main.c) +set_target_properties(maui.link PROPERTIES LINK_FLAGS_RELEASE -s) diff --git a/tools/fastdev/maui.link/main.c b/tools/fastdev/maui.link/main.c new file mode 100644 index 00000000000..d1ae3fc371b --- /dev/null +++ b/tools/fastdev/maui.link/main.c @@ -0,0 +1,320 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#ifndef PATH_MAX +#define PATH_MAX 4096 +#endif + +typedef struct { + long linked; + long unchanged; + long removed; + long errors; +} Counters; + +static void print_help (void); +static int join_path (char *buffer, size_t size, const char *left, const char *right); +static int ensure_dir (const char *path); +static int remove_tree (const char *path); +static int mirror_tree (const char *source_root, const char *target_root, const char *relative_path, Counters *counters); +static int remove_stale_entries (const char *source_root, const char *target_root, const char *relative_path, Counters *counters); +static int link_file (const char *source, const char *target, Counters *counters); + +int main (int argc, char **argv) +{ + if (argc != 3) { + print_help (); + return 1; + } + + const char *source_root = argv [1]; + const char *target_root = argv [2]; + Counters counters = {0, 0, 0, 0}; + + struct stat st; + if (stat (source_root, &st) != 0 || !S_ISDIR (st.st_mode)) { + fprintf (stderr, "error: source directory '%s' does not exist or is not a directory. %s\n", source_root, strerror (errno)); + return 1; + } + + if (ensure_dir (target_root) != 0) { + fprintf (stderr, "error: could not create target directory '%s'. %s\n", target_root, strerror (errno)); + return 1; + } + + int result = 0; + if (mirror_tree (source_root, target_root, "", &counters) != 0) { + result = 1; + } + if (remove_stale_entries (source_root, target_root, "", &counters) != 0) { + result = 1; + } + + printf ("linked [%ld] unchanged [%ld] removed [%ld] errors [%ld]\n", counters.linked, counters.unchanged, counters.removed, counters.errors); + return result; +} + +static void print_help (void) +{ + printf ("maui.link\n"); + printf ("Usage: maui.link \n"); + printf ("\tCreates a symlink mirror of source-directory under target-directory and removes stale target entries.\n"); +} + +static int join_path (char *buffer, size_t size, const char *left, const char *right) +{ + if (right == NULL || right [0] == '\0') { + int n = snprintf (buffer, size, "%s", left); + return n < 0 || (size_t)n >= size ? -1 : 0; + } + + int n = snprintf (buffer, size, "%s/%s", left, right); + return n < 0 || (size_t)n >= size ? -1 : 0; +} + +static int ensure_dir (const char *path) +{ + char tmp [PATH_MAX]; + size_t len; + + if (path == NULL || path [0] == '\0') { + return -1; + } + + if (snprintf (tmp, sizeof (tmp), "%s", path) >= (int)sizeof (tmp)) { + errno = ENAMETOOLONG; + return -1; + } + + len = strlen (tmp); + if (len == 0) { + return -1; + } + + if (tmp [len - 1] == '/') { + tmp [len - 1] = '\0'; + } + + for (char *p = tmp + 1; *p != '\0'; p++) { + if (*p != '/') { + continue; + } + *p = '\0'; + if (mkdir (tmp, 0700) != 0 && errno != EEXIST) { + return -1; + } + *p = '/'; + } + + if (mkdir (tmp, 0700) != 0 && errno != EEXIST) { + return -1; + } + return 0; +} + +static int remove_tree (const char *path) +{ + struct stat st; + if (lstat (path, &st) != 0) { + return errno == ENOENT ? 0 : -1; + } + + if (S_ISDIR (st.st_mode) && !S_ISLNK (st.st_mode)) { + DIR *dir = opendir (path); + if (dir == NULL) { + return -1; + } + + struct dirent *entry; + while ((entry = readdir (dir)) != NULL) { + if (strcmp (entry->d_name, ".") == 0 || strcmp (entry->d_name, "..") == 0) { + continue; + } + + char child [PATH_MAX]; + if (join_path (child, sizeof (child), path, entry->d_name) != 0 || remove_tree (child) != 0) { + closedir (dir); + return -1; + } + } + closedir (dir); + return rmdir (path); + } + + return unlink (path); +} + +static int mirror_tree (const char *source_root, const char *target_root, const char *relative_path, Counters *counters) +{ + char source_dir [PATH_MAX]; + char target_dir [PATH_MAX]; + if (join_path (source_dir, sizeof (source_dir), source_root, relative_path) != 0 || + join_path (target_dir, sizeof (target_dir), target_root, relative_path) != 0) { + fprintf (stderr, "error: path too long while mirroring '%s'\n", relative_path); + counters->errors++; + return -1; + } + + if (ensure_dir (target_dir) != 0) { + fprintf (stderr, "error: could not create directory '%s'. %s\n", target_dir, strerror (errno)); + counters->errors++; + return -1; + } + + DIR *dir = opendir (source_dir); + if (dir == NULL) { + fprintf (stderr, "error: could not open source directory '%s'. %s\n", source_dir, strerror (errno)); + counters->errors++; + return -1; + } + + int result = 0; + struct dirent *entry; + while ((entry = readdir (dir)) != NULL) { + if (strcmp (entry->d_name, ".") == 0 || strcmp (entry->d_name, "..") == 0) { + continue; + } + + char child_relative [PATH_MAX]; + if (join_path (child_relative, sizeof (child_relative), relative_path, entry->d_name) != 0) { + fprintf (stderr, "error: path too long for '%s'\n", entry->d_name); + counters->errors++; + result = -1; + continue; + } + + char source_path [PATH_MAX]; + char target_path [PATH_MAX]; + if (join_path (source_path, sizeof (source_path), source_root, child_relative) != 0 || + join_path (target_path, sizeof (target_path), target_root, child_relative) != 0) { + fprintf (stderr, "error: path too long for '%s'\n", child_relative); + counters->errors++; + result = -1; + continue; + } + + struct stat st; + if (stat (source_path, &st) != 0) { + fprintf (stderr, "error: could not stat source '%s'. %s\n", source_path, strerror (errno)); + counters->errors++; + result = -1; + continue; + } + + if (S_ISDIR (st.st_mode)) { + if (mirror_tree (source_root, target_root, child_relative, counters) != 0) { + result = -1; + } + } else if (S_ISREG (st.st_mode)) { + if (link_file (source_path, target_path, counters) != 0) { + result = -1; + } + } + } + + closedir (dir); + return result; +} + +static int link_file (const char *source, const char *target, Counters *counters) +{ + struct stat st; + if (lstat (target, &st) == 0) { + if (S_ISLNK (st.st_mode)) { + char link_target [PATH_MAX]; + ssize_t len = readlink (target, link_target, sizeof (link_target) - 1); + if (len >= 0) { + link_target [len] = '\0'; + if (strcmp (link_target, source) == 0) { + counters->unchanged++; + return 0; + } + } + } + + if (remove_tree (target) != 0) { + fprintf (stderr, "error: could not remove '%s'. %s\n", target, strerror (errno)); + counters->errors++; + return -1; + } + } else if (errno != ENOENT) { + fprintf (stderr, "error: could not stat target '%s'. %s\n", target, strerror (errno)); + counters->errors++; + return -1; + } + + if (symlink (source, target) != 0) { + fprintf (stderr, "error: could not link '%s' -> '%s'. %s\n", target, source, strerror (errno)); + counters->errors++; + return -1; + } + counters->linked++; + return 0; +} + +static int remove_stale_entries (const char *source_root, const char *target_root, const char *relative_path, Counters *counters) +{ + char target_dir [PATH_MAX]; + if (join_path (target_dir, sizeof (target_dir), target_root, relative_path) != 0) { + fprintf (stderr, "error: path too long while cleaning '%s'\n", relative_path); + counters->errors++; + return -1; + } + + DIR *dir = opendir (target_dir); + if (dir == NULL) { + return errno == ENOENT ? 0 : -1; + } + + int result = 0; + struct dirent *entry; + while ((entry = readdir (dir)) != NULL) { + if (strcmp (entry->d_name, ".") == 0 || strcmp (entry->d_name, "..") == 0) { + continue; + } + + char child_relative [PATH_MAX]; + char source_path [PATH_MAX]; + char target_path [PATH_MAX]; + if (join_path (child_relative, sizeof (child_relative), relative_path, entry->d_name) != 0 || + join_path (source_path, sizeof (source_path), source_root, child_relative) != 0 || + join_path (target_path, sizeof (target_path), target_root, child_relative) != 0) { + fprintf (stderr, "error: path too long while cleaning '%s'\n", entry->d_name); + counters->errors++; + result = -1; + continue; + } + + struct stat source_st; + struct stat target_st; + if (lstat (target_path, &target_st) != 0) { + continue; + } + + if (stat (source_path, &source_st) != 0) { + if (remove_tree (target_path) != 0) { + fprintf (stderr, "error: could not remove stale '%s'. %s\n", target_path, strerror (errno)); + counters->errors++; + result = -1; + } else { + counters->removed++; + } + continue; + } + + if (S_ISDIR (source_st.st_mode) && S_ISDIR (target_st.st_mode) && !S_ISLNK (target_st.st_mode)) { + if (remove_stale_entries (source_root, target_root, child_relative, counters) != 0) { + result = -1; + } + } + } + + closedir (dir); + return result; +} From 2b17e71dde337b3d6b5959c5cf2abc8413fd9caf Mon Sep 17 00:00:00 2001 From: Simon Rozsival Date: Thu, 18 Jun 2026 10:52:46 +0200 Subject: [PATCH 10/12] Drop unused FastDeploy strategy experiments Remove FastDeploy3 and FastDeploy4 after narrowing the experiment to legacy FastDeploy plus the direct-push symlink FastDeploy2 path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tasks/FastDeploy2.cs | 7 +- .../Tasks/FastDeploy3.cs | 254 --------- .../Tasks/FastDeploy4.cs | 483 ------------------ .../Xamarin.Android.Common.Debugging.targets | 61 +-- 4 files changed, 4 insertions(+), 801 deletions(-) delete mode 100644 src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy3.cs delete mode 100644 src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs index fad37ae4e84..4d480a11679 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs @@ -136,10 +136,6 @@ class DiagnosticData { { "deploy.fastdeploy2.stale.remove.ms", "" }, { "deploy.fastdeploy2.override.mkdir.ms", "" }, { "deploy.fastdeploy2.override.copy.ms", "" }, - { "deploy.fastdeploy3.sync.list.ms", "" }, - { "deploy.fastdeploy3.sync.list.files", "" }, - { "deploy.fastdeploy3.override.list.ms", "" }, - { "deploy.fastdeploy3.missing.files", "" }, { "deploy.orchestration.ensure-properties.ms", "" }, { "deploy.orchestration.property-checks.ms", "" }, { "deploy.orchestration.package-check.ms", "" }, @@ -168,6 +164,7 @@ class DiagnosticData { { "deploy.orchestration.terminate.get-pid.ms", "" }, { "deploy.orchestration.terminate.kill.ms", "" }, { "deploy.app.file.transfer.mode", "" }, + { "deploy.fastdeploy2.bulk.batches", "" }, { "deploy.symlink.created.files", "" }, { "deploy.symlink.removed.files", "" }, { "deploy.symlink.tool.ms", "" }, @@ -1264,7 +1261,7 @@ protected async Task UploadFiles (string remoteStagingPath, List "FD3"; - - protected override string RemoteStagingRoot => RemoteDataLocalTmpRoot; - - protected override string GetLocalStagingDirectory () - { - return Path.Combine (GetAndroidProductOutDirectory (), "data", "local", "tmp", "fastdeploy3", PackageName, GetUserId ()); - } - - protected override string GetRemoteAdbPushStagingPath () - { - return $"{RemoteDataLocalTmpRoot}/{PackageName}/{GetUserId ()}"; - } - - protected override async Task DeployFastDevFilesWithAdbPush (string overridePath) - { - string stagingDirectory = GetLocalStagingDirectory (); - var phase = Stopwatch.StartNew (); - var stagedFiles = PrepareAdbPushStagingDirectory (stagingDirectory); - SetDiagnosticElapsed ("deploy.fastdeploy2.local.stage.ms", phase); - if (stagedFiles.Count == 0) { - LogDiagnostic ("No FastDev files were staged for adb sync deployment."); - return true; - } - - string remoteStagingPath = GetRemoteAdbPushStagingPath (); - phase.Restart (); - var mkdirResult = await RunAdbCommand ("shell", "mkdir", "-p", remoteStagingPath); - string output = mkdirResult.Output; - SetDiagnosticElapsed ("deploy.fastdeploy2.remote.mkdir.ms", phase); - if (mkdirResult.ExitCode != 0 || IsShellError (output, "mkdir")) { - LogFastDeploy2Error ("XA0129", output, remoteStagingPath); - return false; - } - - if (!await RemoveStaleRemoteStagingFiles (remoteStagingPath, stagedFiles)) { - return false; - } - - phase.Restart (); - var syncChangedFiles = await GetChangedFilesFromSyncList (remoteStagingPath); - SetDiagnosticElapsed ("deploy.fastdeploy3.sync.list.ms", phase); - if (syncChangedFiles == null) { - return false; - } - SetDiagnosticProperty ("deploy.fastdeploy3.sync.list.files", syncChangedFiles.Count); - - phase.Restart (); - if (!await UploadStagingDirectory (stagingDirectory, remoteStagingPath)) { - return false; - } - SetDiagnosticElapsed ("deploy.fastdeploy2.upload.ms", phase); - - phase.Restart (); - var overrideFiles = await GetOverrideFileList (overridePath); - SetDiagnosticElapsed ("deploy.fastdeploy2.override.stat.ms", phase); - SetDiagnosticElapsed ("deploy.fastdeploy3.override.list.ms", phase); - if (overrideFiles == null) { - return false; - } - - phase.Restart (); - var changedFiles = new HashSet (syncChangedFiles, StringComparer.Ordinal); - int missingFiles = 0; - foreach (var file in stagedFiles) { - if (!overrideFiles.Contains (file) && changedFiles.Add (file)) { - missingFiles++; - } - } - SetDiagnosticElapsed ("deploy.fastdeploy2.compare.ms", phase); - SetDiagnosticProperty ("deploy.fastdeploy3.missing.files", missingFiles); - - if (!await RemoveStaleOverrideFiles (overridePath, stagedFiles, overrideFiles)) { - return false; - } - - return await CopyChangedFiles (remoteStagingPath, overridePath, changedFiles); - } - - protected override async Task UploadStagingDirectory (string stagingDirectory, string remoteStagingPath) - { - var args = new List { "sync" }; - if (!string.IsNullOrEmpty (AdbPushCompressionAlgorithm)) { - args.Add ("-z"); - args.Add (AdbPushCompressionAlgorithm); - } - args.Add ("data"); - - var environmentVariables = new Dictionary { - { "ANDROID_PRODUCT_OUT", GetAndroidProductOutDirectory () }, - }; - var result = await RunAdbCommand (args.ToArray (), environmentVariables); - if (result.ExitCode != 0) { - LogFastDeploy2Error ("XA0129", result.Output, stagingDirectory); - return false; - } - SetAdbPushFileCounts (result.Output); - LogDiagnostic (result.Output); - return true; - } - - async Task> GetChangedFilesFromSyncList (string remoteStagingPath) - { - var args = new [] { "sync", "-l", "data" }; - var environmentVariables = new Dictionary { - { "ANDROID_PRODUCT_OUT", GetAndroidProductOutDirectory () }, - }; - var result = await RunAdbCommand (args, environmentVariables); - if (result.ExitCode != 0) { - LogFastDeploy2Error ("XA0129", result.Output, GetAndroidProductOutDirectory ()); - return null; - } - - var changedFiles = new HashSet (StringComparer.Ordinal); - string prefix = remoteStagingPath.TrimEnd ('/') + "/"; - foreach (string line in result.Output.Split (new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) { - if (!line.StartsWith ("would push:", StringComparison.Ordinal)) { - continue; - } - - int index = line.LastIndexOf (" -> ", StringComparison.Ordinal); - if (index < 0) { - LogDebugMessage ($"Ignoring adb sync -l line '{line}'. Line is incorrectly formatted."); - continue; - } - - string remoteFile = line.Substring (index + 4).Trim (); - if (!remoteFile.StartsWith (prefix, StringComparison.Ordinal)) { - LogDebugMessage ($"Ignoring adb sync -l line '{line}'. Path is outside '{remoteStagingPath}'."); - continue; - } - changedFiles.Add (remoteFile.Substring (prefix.Length)); - } - LogDiagnostic ($"FastDeploy3 adb sync -l listed {changedFiles.Count} changed files."); - return changedFiles; - } - - async Task> GetOverrideFileList (string overridePath) - { - string output = await RunAs ("find", overridePath, "-type", "f"); - if (RaiseRunAsError (output)) { - return null; - } - if (IsMissingDirectoryError (output)) { - return new HashSet (StringComparer.Ordinal); - } - if (IsShellError (output, "find")) { - LogFastDeploy2Error ("XA0129", output, overridePath); - return null; - } - - var files = new HashSet (StringComparer.Ordinal); - string prefix = overridePath.TrimEnd ('/') + "/"; - foreach (string line in output.Split (new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) { - string remoteFile = line.Trim (); - if (!remoteFile.StartsWith (prefix, StringComparison.Ordinal)) { - LogDebugMessage ($"Ignoring override file entry '{line}'. Path is outside '{overridePath}'."); - continue; - } - files.Add (remoteFile.Substring (prefix.Length)); - } - return files; - } - - async Task RemoveStaleOverrideFiles (string overridePath, HashSet stagedFiles, HashSet overrideFiles) - { - var phase = Stopwatch.StartNew (); - var staleFiles = new List (); - foreach (var file in overrideFiles) { - if (!stagedFiles.Contains (file)) { - staleFiles.Add ($"{overridePath}/{file}"); - } - } - - LogDiagnostic ($"FastDeploy3 removing {staleFiles.Count} stale override files."); - SetDiagnosticProperty ("deploy.fastdeploy2.stale.files", staleFiles.Count); - for (int i = 0; i < staleFiles.Count; i += StaleFileRemovalBatchSize) { - var args = new List { "rm", "-f" }; - args.AddRange (staleFiles.Skip (i).Take (StaleFileRemovalBatchSize)); - string output = await RunAs (args.ToArray ()); - if (RaiseRunAsError (output) || IsShellError (output, "rm")) { - LogFastDeploy2Error ("XA0129", output, overridePath); - return false; - } - } - SetDiagnosticElapsed ("deploy.fastdeploy2.stale.remove.ms", phase); - return true; - } - - async Task CopyChangedFiles (string remoteStagingPath, string overridePath, HashSet changedFiles) - { - LogDiagnostic ($"FastDeploy3 copying {changedFiles.Count} changed override files."); - SetDiagnosticProperty ("deploy.fastdeploy2.changed.files", changedFiles.Count); - var filesByDirectory = new Dictionary> (StringComparer.Ordinal); - foreach (string file in changedFiles) { - string directory = Path.GetDirectoryName (file)?.Replace ("\\", "/") ?? ""; - if (!filesByDirectory.TryGetValue (directory, out List files)) { - files = new List (); - filesByDirectory.Add (directory, files); - } - files.Add (file); - } - - var phase = Stopwatch.StartNew (); - foreach (var group in filesByDirectory) { - string targetDirectory = string.IsNullOrEmpty (group.Key) ? overridePath : $"{overridePath}/{group.Key}"; - phase.Restart (); - string output = await RunAs ("mkdir", "-p", targetDirectory); - AddDiagnosticElapsed ("deploy.fastdeploy2.override.mkdir.ms", phase); - if (RaiseRunAsError (output) || IsShellError (output, "mkdir")) { - LogFastDeploy2Error ("XA0129", output, targetDirectory); - return false; - } - - for (int i = 0; i < group.Value.Count; i += CopyBatchSize) { - var args = new List { "cp", "-p" }; - foreach (string file in group.Value.Skip (i).Take (CopyBatchSize)) { - args.Add ($"{remoteStagingPath}/{file}"); - } - args.Add (targetDirectory); - phase.Restart (); - output = await RunAs (args.ToArray ()); - AddDiagnosticElapsed ("deploy.fastdeploy2.override.copy.ms", phase); - if (RaiseRunAsError (output) || IsShellError (output, "cp")) { - LogFastDeploy2Error ("XA0129", output, targetDirectory); - return false; - } - } - } - - return true; - } - - string GetAndroidProductOutDirectory () - { - return Path.Combine (GetFullPath (IntermediateOutputPath), "fastdeploy3-product-out"); - } - } -} diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs deleted file mode 100644 index 9e7d239f227..00000000000 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy4.cs +++ /dev/null @@ -1,483 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -using Microsoft.Android.Build.Tasks; - -namespace Xamarin.Android.Tasks -{ - public class FastDeploy4 : FastDeploy2 - { - const string RemoteStagingRootPath = "/tmp/fastdeploy4"; - const int CopyBatchSize = 25; - const int RemoveBatchSize = 100; - const int MaxAdbCommandLength = 4096; - - public override string TaskPrefix => "FD4"; - - public string PushMode { get; set; } = "SingleFile"; - - protected override string RemoteStagingRoot => RemoteStagingRootPath; - - protected override async Task DeployFastDevFilesWithAdbPush (string overridePath) - { - string remoteStagingPath = GetRemoteAdbPushStagingPath (); - var phase = Stopwatch.StartNew (); - var files = PrepareDirectPushFiles (); - var expectedFiles = new HashSet (files.Select (file => file.RelativePath), StringComparer.Ordinal); - SetDiagnosticElapsed ("deploy.fastdeploy2.local.stage.ms", phase); - if (files.Count == 0) { - LogDiagnostic ("No FastDev files were prepared for direct adb push deployment."); - return true; - } - - phase.Restart (); - if (!await CreateRemoteStagingDirectories (remoteStagingPath, files)) { - return false; - } - SetDiagnosticElapsed ("deploy.fastdeploy2.remote.mkdir.ms", phase); - - if (!await RemoveStaleRemoteStagingFiles (remoteStagingPath, expectedFiles)) { - return false; - } - - var pushMode = PushMode ?? ""; - HashSet changedFiles; - if (string.Equals (pushMode, "SingleFile", StringComparison.OrdinalIgnoreCase)) { - phase.Restart (); - changedFiles = await PushFilesOneByOne (remoteStagingPath, files); - SetDiagnosticElapsed ("deploy.fastdeploy2.upload.ms", phase); - } else if (string.Equals (pushMode, "Bulk", StringComparison.OrdinalIgnoreCase)) { - phase.Restart (); - if (!await PushFilesInBulk (remoteStagingPath, files)) { - return false; - } - SetDiagnosticElapsed ("deploy.fastdeploy2.upload.ms", phase); - - phase.Restart (); - var stagedFileData = await GetRemoteFileData (remoteStagingPath, runAs: false); - SetDiagnosticElapsed ("deploy.fastdeploy2.staging.stat.ms", phase); - if (stagedFileData == null) { - return false; - } - - phase.Restart (); - var overrideFileData = await GetRemoteFileData (overridePath, runAs: true); - SetDiagnosticElapsed ("deploy.fastdeploy2.override.stat.ms", phase); - if (overrideFileData == null) { - return false; - } - - if (!await RemoveStaleOverrideFiles (overridePath, stagedFileData.Keys, overrideFileData.Keys)) { - return false; - } - - phase.Restart (); - changedFiles = GetChangedFiles (stagedFileData, overrideFileData); - SetDiagnosticElapsed ("deploy.fastdeploy2.compare.ms", phase); - } else { - LogFastDeploy2Error ("XA0129", $"Invalid FastDeploy4 PushMode '{PushMode}'. Supported values are 'SingleFile' and 'Bulk'.", PushMode); - return false; - } - - SetDiagnosticProperty ("deploy.fastdeploy4.push.mode", pushMode); - - phase.Restart (); - var overrideFiles = await GetOverrideFileList (overridePath); - SetDiagnosticElapsed ("deploy.fastdeploy3.override.list.ms", phase); - if (overrideFiles == null) { - return false; - } - - phase.Restart (); - int missingFiles = 0; - foreach (string file in expectedFiles) { - if (!overrideFiles.Contains (file) && changedFiles.Add (file)) { - missingFiles++; - } - } - SetDiagnosticElapsed ("deploy.fastdeploy2.compare.ms", phase); - SetDiagnosticProperty ("deploy.fastdeploy3.missing.files", missingFiles); - - if (!await RemoveStaleOverrideFiles (overridePath, expectedFiles, overrideFiles)) { - return false; - } - - return await CopyChangedFiles (remoteStagingPath, overridePath, changedFiles); - } - - new List PrepareDirectPushFiles () - { - var files = new List (); - foreach (var file in FastDevFiles ?? Array.Empty ()) { - if (Path.GetExtension (file.ItemSpec) == ".so") { - string abi = AndroidRidAbiHelper.GetNativeLibraryAbi (file); - if (abi != PrimaryCpuAbi) { - LogDebugMessage ($"NotifySync SkipCopyFile {file.ItemSpec} abi not suitable for this device."); - continue; - } - } - - files.Add (new DirectPushFile { - LocalPath = file.ItemSpec, - RelativePath = GetAdbPushTargetPath (file), - }); - LogDiagnostic ($"Prepared {file.ItemSpec} => {files [files.Count - 1].RelativePath}"); - } - - if (EnvironmentFiles?.Length > 0) { - byte [] environmentData = CreateEnvironmentFileData (EnvironmentFiles, out DateTime newestFileDateTime); - if (environmentData.Length > 0) { - string environmentFile = Path.Combine (GetFullPath (IntermediateOutputPath), "fastdeploy4", PrimaryCpuAbi, "environment"); - WriteFileIfChanged (environmentFile, environmentData, newestFileDateTime); - files.Add (new DirectPushFile { - LocalPath = environmentFile, - RelativePath = $"{PrimaryCpuAbi}/environment", - }); - } - } - - return files; - } - - async Task CreateRemoteStagingDirectories (string remoteStagingPath, List files) - { - var directories = new HashSet (StringComparer.Ordinal) { remoteStagingPath }; - foreach (var file in files) { - string directory = Path.GetDirectoryName (file.RelativePath)?.Replace ("\\", "/") ?? ""; - if (!string.IsNullOrEmpty (directory)) { - directories.Add ($"{remoteStagingPath}/{directory}"); - } - } - - foreach (var batch in BatchArguments ("mkdir", "-p", directories)) { - var result = await RunAdbCommand (new [] { "shell" }.Concat (batch).ToArray ()); - if (result.ExitCode != 0 || IsShellError (result.Output, "mkdir")) { - LogFastDeploy2Error ("XA0129", result.Output, remoteStagingPath); - return false; - } - } - return true; - } - - async Task> PushFilesOneByOne (string remoteStagingPath, List files) - { - var changedFiles = new HashSet (StringComparer.Ordinal); - int pushed = 0; - int skipped = 0; - foreach (var file in files) { - var args = CreatePushArgs (file.LocalPath, $"{remoteStagingPath}/{file.RelativePath}"); - var result = await RunAdbCommand (args.ToArray ()); - if (result.ExitCode != 0) { - LogFastDeploy2Error ("XA0129", result.Output, file.LocalPath); - return null; - } - var counts = TryParsePushSummary (result.Output); - pushed += counts.pushed; - skipped += counts.skipped; - if (counts.pushed > 0) { - changedFiles.Add (file.RelativePath); - } - LogDiagnostic (result.Output); - } - SetDiagnosticProperty ("deploy.fastdeploy2.adb.pushed.files", pushed); - SetDiagnosticProperty ("deploy.fastdeploy2.adb.skipped.files", skipped); - SetDiagnosticProperty ("deploy.fastdeploy4.direct.push.files", files.Count); - return changedFiles; - } - - async Task PushFilesInBulk (string remoteStagingPath, List files) - { - int pushed = 0; - int skipped = 0; - int batches = 0; - foreach (var group in files.GroupBy (file => Path.GetDirectoryName (file.RelativePath)?.Replace ("\\", "/") ?? "", StringComparer.Ordinal)) { - string remoteDirectory = string.IsNullOrEmpty (group.Key) ? remoteStagingPath : $"{remoteStagingPath}/{group.Key}"; - foreach (var batch in BatchPushFiles (group.ToList (), remoteDirectory)) { - var result = await RunAdbCommand (batch.ToArray ()); - if (result.ExitCode != 0) { - LogFastDeploy2Error ("XA0129", result.Output, remoteDirectory); - return false; - } - var counts = TryParsePushSummary (result.Output); - pushed += counts.pushed; - skipped += counts.skipped; - batches++; - LogDiagnostic (result.Output); - } - } - SetDiagnosticProperty ("deploy.fastdeploy2.adb.pushed.files", pushed); - SetDiagnosticProperty ("deploy.fastdeploy2.adb.skipped.files", skipped); - SetDiagnosticProperty ("deploy.fastdeploy4.bulk.batches", batches); - return true; - } - - HashSet GetChangedFiles (Dictionary stagedFiles, Dictionary overrideFiles) - { - var changedFiles = new HashSet (StringComparer.Ordinal); - foreach (var file in stagedFiles) { - if (!overrideFiles.TryGetValue (file.Key, out RemoteFileInfo existing) || - existing.Size != file.Value.Size || - existing.ModifiedTime != file.Value.ModifiedTime) { - changedFiles.Add (file.Key); - } - } - return changedFiles; - } - - async Task> GetOverrideFileList (string overridePath) - { - string output = await RunAs ("find", overridePath, "-type", "f"); - if (RaiseRunAsError (output)) { - return null; - } - if (IsMissingDirectoryError (output)) { - return new HashSet (StringComparer.Ordinal); - } - if (IsShellError (output, "find")) { - LogFastDeploy2Error ("XA0129", output, overridePath); - return null; - } - - var files = new HashSet (StringComparer.Ordinal); - string prefix = overridePath.TrimEnd ('/') + "/"; - foreach (string line in output.Split (new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) { - string remoteFile = line.Trim (); - if (remoteFile.StartsWith (prefix, StringComparison.Ordinal)) { - files.Add (remoteFile.Substring (prefix.Length)); - } - } - return files; - } - - async Task RemoveStaleOverrideFiles (string overridePath, IEnumerable stagedFiles, IEnumerable overrideFiles) - { - var phase = Stopwatch.StartNew (); - var staged = new HashSet (stagedFiles, StringComparer.Ordinal); - var staleFiles = new List (); - foreach (var file in overrideFiles) { - if (!staged.Contains (file)) { - staleFiles.Add ($"{overridePath}/{file}"); - } - } - - SetDiagnosticProperty ("deploy.fastdeploy2.stale.files", staleFiles.Count); - foreach (var batch in BatchArguments ("rm", "-f", staleFiles)) { - string output = await RunAs (batch.ToArray ()); - if (RaiseRunAsError (output) || IsShellError (output, "rm")) { - LogFastDeploy2Error ("XA0129", output, overridePath); - return false; - } - } - SetDiagnosticElapsed ("deploy.fastdeploy2.stale.remove.ms", phase); - return true; - } - - async Task CopyChangedFiles (string remoteStagingPath, string overridePath, HashSet changedFiles) - { - SetDiagnosticProperty ("deploy.fastdeploy2.changed.files", changedFiles.Count); - var filesByDirectory = new Dictionary> (StringComparer.Ordinal); - foreach (string file in changedFiles) { - string directory = Path.GetDirectoryName (file)?.Replace ("\\", "/") ?? ""; - if (!filesByDirectory.TryGetValue (directory, out List files)) { - files = new List (); - filesByDirectory.Add (directory, files); - } - files.Add (file); - } - - var phase = Stopwatch.StartNew (); - foreach (var group in filesByDirectory) { - string targetDirectory = string.IsNullOrEmpty (group.Key) ? overridePath : $"{overridePath}/{group.Key}"; - phase.Restart (); - string output = await RunAs ("mkdir", "-p", targetDirectory); - AddDiagnosticElapsed ("deploy.fastdeploy2.override.mkdir.ms", phase); - if (RaiseRunAsError (output) || IsShellError (output, "mkdir")) { - LogFastDeploy2Error ("XA0129", output, targetDirectory); - return false; - } - - for (int i = 0; i < group.Value.Count; i += CopyBatchSize) { - var args = new List { "cp", "-p" }; - foreach (string file in group.Value.Skip (i).Take (CopyBatchSize)) { - args.Add ($"{remoteStagingPath}/{file}"); - } - args.Add (targetDirectory); - phase.Restart (); - output = await RunAs (args.ToArray ()); - AddDiagnosticElapsed ("deploy.fastdeploy2.override.copy.ms", phase); - if (RaiseRunAsError (output) || IsShellError (output, "cp")) { - LogFastDeploy2Error ("XA0129", output, targetDirectory); - return false; - } - } - } - return true; - } - - List CreatePushArgs (string localPath, string remotePath) - { - var args = new List { "push" }; - if (!string.IsNullOrEmpty (AdbPushCompressionAlgorithm)) { - args.Add ("-z"); - args.Add (AdbPushCompressionAlgorithm); - } - args.Add ("--sync"); - args.Add (localPath); - args.Add (remotePath); - return args; - } - - IEnumerable> BatchPushFiles (List files, string remoteDirectory) - { - var batch = CreatePushArgsPrefix (); - int length = EstimateCommandLength (batch) + remoteDirectory.Length + 4; - foreach (var file in files) { - if (Path.GetFileName (file.LocalPath) != Path.GetFileName (file.RelativePath)) { - var single = CreatePushArgs (file.LocalPath, $"{remoteDirectory}/{Path.GetFileName (file.RelativePath)}"); - yield return single; - continue; - } - - int itemLength = file.LocalPath.Length + 3; - if (batch.Count > 3 && length + itemLength >= MaxAdbCommandLength) { - batch.Add (remoteDirectory); - yield return batch; - batch = CreatePushArgsPrefix (); - length = EstimateCommandLength (batch) + remoteDirectory.Length + 4; - } - batch.Add (file.LocalPath); - length += itemLength; - } - if (batch.Count > 3) { - batch.Add (remoteDirectory); - yield return batch; - } - } - - List CreatePushArgsPrefix () - { - var args = new List { "push" }; - if (!string.IsNullOrEmpty (AdbPushCompressionAlgorithm)) { - args.Add ("-z"); - args.Add (AdbPushCompressionAlgorithm); - } - args.Add ("--sync"); - return args; - } - - IEnumerable> BatchArguments (string command, string option, IEnumerable values) - { - var batch = new List { command, option }; - int length = command.Length + option.Length + 2; - foreach (var value in values) { - int itemLength = value.Length + 3; - if (batch.Count > 2 && length + itemLength >= MaxAdbCommandLength) { - yield return batch; - batch = new List { command, option }; - length = command.Length + option.Length + 2; - } - batch.Add (value); - length += itemLength; - } - if (batch.Count > 2) { - yield return batch; - } - } - - int EstimateCommandLength (List args) - { - int length = 0; - foreach (var arg in args) { - length += arg.Length + 3; - } - return length; - } - - (int pushed, int skipped) TryParsePushSummary (string output) - { - int pushed = 0; - int skipped = 0; - foreach (var line in output.Split (new char [] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries)) { - var match = System.Text.RegularExpressions.Regex.Match (line, @"(?\d+) files? pushed, (?\d+) skipped"); - if (!match.Success) { - continue; - } - pushed = int.Parse (match.Groups ["pushed"].Value); - skipped = int.Parse (match.Groups ["skipped"].Value); - } - return (pushed, skipped); - } - - string GetAdbPushTargetPath (Microsoft.Build.Framework.ITaskItem file) - { - string targetPath = file.GetMetadata ("TargetPath"); - if (string.IsNullOrEmpty (targetPath)) { - LogDiagnostic ($"'TargetPath' meta data not found on '{file.ItemSpec}'. Falling back to'DestinationSubPath'"); - targetPath = file.GetMetadata ("DestinationSubPath"); - } - if (!string.IsNullOrEmpty (targetPath)) { - return targetPath.Replace ("\\", "/"); - } - return Path.GetFileName (file.ItemSpec); - } - - byte [] CreateEnvironmentFileData (Microsoft.Build.Framework.ITaskItem [] environments, out DateTime newestFileDateTime) - { - int maxKeyLength = 0; - int maxValueLength = 0; - newestFileDateTime = DateTime.MinValue; - var data = new Dictionary (); - foreach (var env in environments ?? Array.Empty ()) { - if (!File.Exists (env.ItemSpec)) - continue; - DateTime modifiedDateTime = File.GetLastWriteTimeUtc (env.ItemSpec); - if (modifiedDateTime > newestFileDateTime) - newestFileDateTime = modifiedDateTime; - foreach (string line in File.ReadLines (env.ItemSpec)) { - if (string.IsNullOrEmpty (line)) - continue; - int index = line.IndexOf ('='); - if (index == -1) { - LogDebugMessage ($"Skipping invalid environment line: {line}"); - continue; - } - var key = line.Substring (0, index); - var value = line.Substring (index + 1); - maxKeyLength = Math.Max (maxKeyLength, key.Length); - maxValueLength = Math.Max (maxValueLength, value.Length); - data [key] = value; - } - } - - if (newestFileDateTime == DateTime.MinValue) { - return Array.Empty (); - } - - maxKeyLength++; - maxValueLength++; - - using (var stream = new MemoryStream ()) - using (var binaryWriter = new BinaryWriter (stream, Encoding.ASCII)) { - binaryWriter.Write (Encoding.ASCII.GetBytes ("0x" + maxKeyLength.ToString ("X8") + '\0')); - binaryWriter.Write (Encoding.ASCII.GetBytes ("0x" + maxValueLength.ToString ("X8") + '\0')); - foreach (var kvp in data) { - binaryWriter.Write (Encoding.ASCII.GetBytes (kvp.Key.PadRight (maxKeyLength, '\0'))); - binaryWriter.Write (Encoding.ASCII.GetBytes (kvp.Value.PadRight (maxValueLength, '\0'))); - } - binaryWriter.Flush (); - return stream.ToArray (); - } - } - - new class DirectPushFile - { - public string LocalPath { get; set; } - public string RelativePath { get; set; } - } - } -} diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets index 5f64b1f6ffc..c67683a0482 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets @@ -24,8 +24,6 @@ Copyright (C) 2016 Xamarin. All rights reserved. - - @@ -325,17 +323,13 @@ Copyright (C) 2016 Xamarin. All rights reserved. <_AndroidIsTestOnlyPackage Condition=" '$(_AndroidIsTestOnlyPackage)' == '' ">False <_FastDeploymentDiagnosticLogging Condition=" '$(_FastDeploymentDiagnosticLogging)' == '' ">False <_AndroidFastDevStrategy Condition=" '$(_AndroidFastDevStrategy)' == '' ">FastDeploy - <_AndroidFastDeploy4PushMode Condition=" '$(_AndroidFastDeploy4PushMode)' == '' ">SingleFile <_AndroidFastDeployAppFileTransferMode Condition=" '$(_AndroidFastDeployAppFileTransferMode)' == '' And '$(_AndroidFastDevStrategy)' == 'FastDeploy2' ">Symlink <_AndroidFastDeployAppFileTransferMode Condition=" '$(_AndroidFastDeployAppFileTransferMode)' == '' ">Copy any - + Condition=" '$(_AndroidFastDevStrategy)' != 'FastDeploy' And '$(_AndroidFastDevStrategy)' != 'FastDeploy2' " + Text="Invalid _AndroidFastDevStrategy value '$(_AndroidFastDevStrategy)'. Supported values are 'FastDeploy' and 'FastDeploy2'." /> @@ -393,57 +387,6 @@ Copyright (C) 2016 Xamarin. All rights reserved. IntermediateOutputPath="$(IntermediateOutputPath)" EnvironmentFiles="@(_EnvironmentFiles)" /> - - Date: Thu, 18 Jun 2026 11:01:14 +0200 Subject: [PATCH 11/12] Add manifest-driven FastDeploy5 strategy Add an experimental FastDeploy5 strategy which compares a local manifest, pushes only changed files without adb --sync, removes stale staged files, and mirrors the remote staging tree through maui.link. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tasks/FastDeploy2.cs | 12 +- .../Tasks/FastDeploy5.cs | 306 ++++++++++++++++++ .../Xamarin.Android.Common.Debugging.targets | 31 +- 3 files changed, 341 insertions(+), 8 deletions(-) create mode 100644 src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy5.cs diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs index 4d480a11679..2a9fa6ebce5 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs @@ -610,7 +610,7 @@ protected virtual async Task DeployFastDevFilesWithAdbPush (string overrid return await CopyChangedFiles (remoteStagingPath, overridePath, stagedFileData, overrideFileData); } - async Task CreateRemoteStagingDirectories (string remoteStagingPath, HashSet stagedFiles) + protected async Task CreateRemoteStagingDirectories (string remoteStagingPath, HashSet stagedFiles) { var directories = new HashSet (StringComparer.Ordinal) { remoteStagingPath }; foreach (var file in stagedFiles) { @@ -935,7 +935,7 @@ Dictionary ParseRemoteFileData (string rootPath, string return files; } - async Task RemoveStaleOverrideFiles (string overridePath, Dictionary stagedFiles, Dictionary overrideFiles) + protected async Task RemoveStaleOverrideFiles (string overridePath, Dictionary stagedFiles, Dictionary overrideFiles) { var phase = Stopwatch.StartNew (); var staleFiles = new List (); @@ -960,7 +960,7 @@ async Task RemoveStaleOverrideFiles (string overridePath, Dictionary UpdateOverrideSymlinks (string remoteStagingPath, string overridePath, HashSet stagedFiles) + protected async Task UpdateOverrideSymlinks (string remoteStagingPath, string overridePath, HashSet stagedFiles) { var toolInstallPhase = Stopwatch.StartNew (); bool linkToolInstalled = await InstallLinkTool (); @@ -1117,7 +1117,7 @@ async Task CreateOverrideSymlinks (string remoteStagingPath, string overri return true; } - async Task CopyChangedFiles (string remoteStagingPath, string overridePath, Dictionary stagedFiles, Dictionary overrideFiles) + protected async Task CopyChangedFiles (string remoteStagingPath, string overridePath, Dictionary stagedFiles, Dictionary overrideFiles) { var phase = Stopwatch.StartNew (); var changedFiles = new List (); @@ -1171,7 +1171,7 @@ async Task CopyChangedFiles (string remoteStagingPath, string overridePath return true; } - IEnumerable> BatchArguments (string command, string option, IEnumerable values) + protected IEnumerable> BatchArguments (string command, string option, IEnumerable values) { var batch = new List { command, option }; int length = command.Length + option.Length + 2; @@ -1319,7 +1319,7 @@ int EstimateCommandLength (List args) return length; } - (int pushed, int skipped) TryParsePushSummary (string output) + protected (int pushed, int skipped) TryParsePushSummary (string output) { int pushed = 0; int skipped = 0; diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy5.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy5.cs new file mode 100644 index 00000000000..57676abf07a --- /dev/null +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy5.cs @@ -0,0 +1,306 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; + +namespace Xamarin.Android.Tasks +{ + public class FastDeploy5 : FastDeploy2 + { + const string RemoteStagingRootPath = "/tmp/fastdeploy5"; + const string RemoteReadyMarker = ".fastdeploy5-ready"; + const int MaxAdbCommandLength = 4096; + + public override string TaskPrefix => "FD5"; + + protected override string RemoteStagingRoot => RemoteStagingRootPath; + + protected override async Task DeployFastDevFilesWithAdbPush (string overridePath) + { + var phase = Stopwatch.StartNew (); + var files = PrepareDirectPushFiles (); + var expectedFiles = new HashSet (files.Select (file => file.RelativePath), StringComparer.Ordinal); + var currentManifest = CreateManifest (files); + SetDiagnosticElapsed ("deploy.fastdeploy2.local.stage.ms", phase); + if (files.Count == 0) { + LogDiagnostic ("No FastDev files were prepared for manifest adb push deployment."); + return true; + } + + string remoteStagingPath = GetRemoteAdbPushStagingPath (); + bool remoteReady = await IsRemoteReady (remoteStagingPath); + var previousManifest = remoteReady ? LoadPreviousManifest () : null; + if (previousManifest == null) { + SetDiagnosticProperty ("deploy.fastdeploy5.full.push", 1); + } + + var changedFiles = GetChangedFiles (currentManifest, previousManifest); + var removedFiles = GetRemovedFiles (currentManifest, previousManifest); + SetDiagnosticProperty ("deploy.fastdeploy5.manifest.changed.files", changedFiles.Count); + SetDiagnosticProperty ("deploy.fastdeploy5.manifest.removed.files", removedFiles.Count); + + phase.Restart (); + string output = await CreateRemoteStagingDirectories (remoteStagingPath, expectedFiles); + SetDiagnosticElapsed ("deploy.fastdeploy2.remote.mkdir.ms", phase); + if (!string.IsNullOrEmpty (output) && IsShellError (output, "mkdir")) { + LogFastDeploy2Error ("XA0129", output, remoteStagingPath); + return false; + } + + phase.Restart (); + if (!await RemoveRemoteStaleFiles (remoteStagingPath, removedFiles)) { + return false; + } + SetDiagnosticElapsed ("deploy.fastdeploy2.remote.staging.cleanup.ms", phase); + + phase.Restart (); + if (!await UploadChangedFiles (remoteStagingPath, files, changedFiles)) { + return false; + } + SetDiagnosticElapsed ("deploy.fastdeploy2.upload.ms", phase); + + bool result; + if (UseSymlinkAppFileTransfer ()) { + result = await UpdateOverrideSymlinks (remoteStagingPath, overridePath, expectedFiles); + } else { + phase.Restart (); + var stagedFileData = await GetRemoteFileData (remoteStagingPath, runAs: false); + SetDiagnosticElapsed ("deploy.fastdeploy2.staging.stat.ms", phase); + if (stagedFileData == null) { + return false; + } + + phase.Restart (); + var overrideFileData = await GetRemoteFileData (overridePath, runAs: true); + SetDiagnosticElapsed ("deploy.fastdeploy2.override.stat.ms", phase); + if (overrideFileData == null) { + return false; + } + + if (!await RemoveStaleOverrideFiles (overridePath, stagedFileData, overrideFileData)) { + return false; + } + + result = await CopyChangedFiles (remoteStagingPath, overridePath, stagedFileData, overrideFileData); + } + + if (result) { + WriteManifest (currentManifest); + await MarkRemoteReady (remoteStagingPath); + } + return result; + } + + Dictionary CreateManifest (List files) + { + var manifest = new Dictionary (StringComparer.Ordinal); + foreach (var file in files) { + var info = new FileInfo (file.LocalPath); + manifest [file.RelativePath] = new ManifestEntry { + RelativePath = file.RelativePath, + LocalPath = file.LocalPath, + Size = info.Length, + LastWriteTimeUtcTicks = info.LastWriteTimeUtc.Ticks, + }; + } + return manifest; + } + + HashSet GetChangedFiles (Dictionary currentManifest, Dictionary previousManifest) + { + if (previousManifest == null) { + return new HashSet (currentManifest.Keys, StringComparer.Ordinal); + } + + var changedFiles = new HashSet (StringComparer.Ordinal); + foreach (var entry in currentManifest) { + if (!previousManifest.TryGetValue (entry.Key, out ManifestEntry previous) || + previous.Size != entry.Value.Size || + previous.LastWriteTimeUtcTicks != entry.Value.LastWriteTimeUtcTicks) { + changedFiles.Add (entry.Key); + } + } + return changedFiles; + } + + List GetRemovedFiles (Dictionary currentManifest, Dictionary previousManifest) + { + var removedFiles = new List (); + if (previousManifest == null) { + return removedFiles; + } + + foreach (var entry in previousManifest.Keys) { + if (!currentManifest.ContainsKey (entry)) { + removedFiles.Add (entry); + } + } + return removedFiles; + } + + async Task UploadChangedFiles (string remoteStagingPath, List files, HashSet changedFiles) + { + int pushed = 0; + int skipped = 0; + int batches = 0; + var changedFileList = files.Where (file => changedFiles.Contains (file.RelativePath)).ToList (); + foreach (var group in changedFileList.GroupBy (file => Path.GetDirectoryName (file.RelativePath)?.Replace ("\\", "/") ?? "", StringComparer.Ordinal)) { + string remoteDirectory = string.IsNullOrEmpty (group.Key) ? remoteStagingPath : $"{remoteStagingPath}/{group.Key}"; + foreach (var batch in BatchPushFilesWithoutSync (group.ToList (), remoteDirectory)) { + var result = await RunAdbCommand (batch.ToArray ()); + if (result.ExitCode != 0) { + LogFastDeploy2Error ("XA0129", result.Output, remoteDirectory); + return false; + } + var counts = TryParsePushSummary (result.Output); + pushed += counts.pushed; + skipped += counts.skipped; + batches++; + LogDiagnostic (result.Output); + } + } + SetDiagnosticProperty ("deploy.fastdeploy2.adb.pushed.files", pushed); + SetDiagnosticProperty ("deploy.fastdeploy2.adb.skipped.files", skipped); + SetDiagnosticProperty ("deploy.fastdeploy2.bulk.batches", batches); + SetDiagnosticProperty ("deploy.fastdeploy2.changed.files", changedFiles.Count); + return true; + } + + async Task RemoveRemoteStaleFiles (string remoteStagingPath, List removedFiles) + { + foreach (var batch in BatchArguments ("rm", "-f", removedFiles.Select (file => $"{remoteStagingPath}/{file}"))) { + var args = new [] { "shell" }.Concat (batch).ToArray (); + var result = await RunAdbCommand (args); + if (result.ExitCode != 0 || IsShellError (result.Output, "rm")) { + LogFastDeploy2Error ("XA0129", result.Output, remoteStagingPath); + return false; + } + } + return true; + } + + IEnumerable> BatchPushFilesWithoutSync (List files, string remoteDirectory) + { + var batch = CreatePushArgsPrefix (); + int length = EstimateCommandLength (batch) + remoteDirectory.Length + 4; + foreach (var file in files) { + if (Path.GetFileName (file.LocalPath) != Path.GetFileName (file.RelativePath)) { + yield return CreatePushArgs (file.LocalPath, $"{remoteDirectory}/{Path.GetFileName (file.RelativePath)}"); + continue; + } + + int itemLength = file.LocalPath.Length + 3; + if (batch.Count > 1 && length + itemLength >= MaxAdbCommandLength) { + batch.Add (remoteDirectory); + yield return batch; + batch = CreatePushArgsPrefix (); + length = EstimateCommandLength (batch) + remoteDirectory.Length + 4; + } + batch.Add (file.LocalPath); + length += itemLength; + } + if (batch.Count > 1) { + batch.Add (remoteDirectory); + yield return batch; + } + } + + List CreatePushArgs (string localPath, string remotePath) + { + var args = CreatePushArgsPrefix (); + args.Add (localPath); + args.Add (remotePath); + return args; + } + + List CreatePushArgsPrefix () + { + var args = new List { "push" }; + if (!string.IsNullOrEmpty (AdbPushCompressionAlgorithm)) { + args.Add ("-z"); + args.Add (AdbPushCompressionAlgorithm); + } + return args; + } + + int EstimateCommandLength (List args) + { + int length = 0; + foreach (var arg in args) { + length += arg.Length + 3; + } + return length; + } + + async Task IsRemoteReady (string remoteStagingPath) + { + var result = await RunAdbCommand ("shell", "test", "-f", $"{remoteStagingPath}/{RemoteReadyMarker}"); + return result.ExitCode == 0; + } + + async Task MarkRemoteReady (string remoteStagingPath) + { + await RunAdbCommand ("shell", "touch", $"{remoteStagingPath}/{RemoteReadyMarker}"); + } + + Dictionary LoadPreviousManifest () + { + string manifestFile = GetManifestFilePath (); + if (!File.Exists (manifestFile)) { + return null; + } + + try { + var manifest = JsonSerializer.Deserialize> (File.ReadAllText (manifestFile)); + return manifest == null ? null : new Dictionary (manifest, StringComparer.Ordinal); + } catch (Exception ex) { + LogDiagnostic ($"Ignoring FastDeploy5 manifest '{manifestFile}'. {ex}"); + return null; + } + } + + void WriteManifest (Dictionary manifest) + { + string manifestFile = GetManifestFilePath (); + Directory.CreateDirectory (Path.GetDirectoryName (manifestFile)); + File.WriteAllText (manifestFile, JsonSerializer.Serialize (manifest, new JsonSerializerOptions { WriteIndented = true })); + } + + string GetManifestFilePath () + { + return Path.Combine (GetFullPath (IntermediateOutputPath), "fastdeploy5", GetSafeFileName (PackageName), GetSafeFileName (GetUserId ()), GetSafeFileName (PrimaryCpuAbi), "manifest.json"); + } + + static string GetSafeFileName (string value) + { + if (string.IsNullOrEmpty (value)) { + return "_"; + } + + var builder = new StringBuilder (value.Length); + foreach (char c in value) { + builder.Append (char.IsLetterOrDigit (c) || c == '.' || c == '-' || c == '_' ? c : '_'); + } + return builder.ToString (); + } + + class ManifestEntry { + [JsonPropertyName ("relativePath")] + public string RelativePath { get; set; } + + [JsonPropertyName ("localPath")] + public string LocalPath { get; set; } + + [JsonPropertyName ("size")] + public long Size { get; set; } + + [JsonPropertyName ("lastWriteTimeUtcTicks")] + public long LastWriteTimeUtcTicks { get; set; } + } + } +} diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets index c67683a0482..ac07788a064 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets @@ -24,6 +24,7 @@ Copyright (C) 2016 Xamarin. All rights reserved. + @@ -324,12 +325,13 @@ Copyright (C) 2016 Xamarin. All rights reserved. <_FastDeploymentDiagnosticLogging Condition=" '$(_FastDeploymentDiagnosticLogging)' == '' ">False <_AndroidFastDevStrategy Condition=" '$(_AndroidFastDevStrategy)' == '' ">FastDeploy <_AndroidFastDeployAppFileTransferMode Condition=" '$(_AndroidFastDeployAppFileTransferMode)' == '' And '$(_AndroidFastDevStrategy)' == 'FastDeploy2' ">Symlink + <_AndroidFastDeployAppFileTransferMode Condition=" '$(_AndroidFastDeployAppFileTransferMode)' == '' And '$(_AndroidFastDevStrategy)' == 'FastDeploy5' ">Symlink <_AndroidFastDeployAppFileTransferMode Condition=" '$(_AndroidFastDeployAppFileTransferMode)' == '' ">Copy any + Condition=" '$(_AndroidFastDevStrategy)' != 'FastDeploy' And '$(_AndroidFastDevStrategy)' != 'FastDeploy2' And '$(_AndroidFastDevStrategy)' != 'FastDeploy5' " + Text="Invalid _AndroidFastDevStrategy value '$(_AndroidFastDevStrategy)'. Supported values are 'FastDeploy', 'FastDeploy2', and 'FastDeploy5'." /> @@ -387,6 +389,31 @@ Copyright (C) 2016 Xamarin. All rights reserved. IntermediateOutputPath="$(IntermediateOutputPath)" EnvironmentFiles="@(_EnvironmentFiles)" /> + Date: Thu, 18 Jun 2026 11:16:56 +0200 Subject: [PATCH 12/12] Add shell symlink mode for FastDeploy5 Add a FastDeploy5 mode which uses manifest data to run rm and ln -sf together in batched run-as shell commands, avoiding the maui.link helper for symlink maintenance. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Tasks/FastDeploy2.cs | 1 + .../Tasks/FastDeploy5.cs | 213 ++++++++++++++++-- .../Xamarin.Android.Common.Debugging.targets | 4 +- 3 files changed, 201 insertions(+), 17 deletions(-) diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs index 2a9fa6ebce5..5c76ca26b78 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy2.cs @@ -170,6 +170,7 @@ class DiagnosticData { { "deploy.symlink.tool.ms", "" }, { "deploy.symlink.tool.result", "" }, { "deploy.symlink.tool.install.ms", "" }, + { "deploy.symlink.shell.update.ms", "" }, { "pii.deploy.error", "" }, { "pii.deploy.file", "" }, }; diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy5.cs b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy5.cs index 57676abf07a..653baa37c05 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy5.cs +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Tasks/FastDeploy5.cs @@ -65,35 +65,218 @@ protected override async Task DeployFastDevFilesWithAdbPush (string overri SetDiagnosticElapsed ("deploy.fastdeploy2.upload.ms", phase); bool result; - if (UseSymlinkAppFileTransfer ()) { + if (UseShellSymlinkAppFileTransfer ()) { + result = await UpdateOverrideShellSymlinks (remoteStagingPath, overridePath, currentManifest, previousManifest, removedFiles); + } else if (UseSymlinkAppFileTransfer ()) { result = await UpdateOverrideSymlinks (remoteStagingPath, overridePath, expectedFiles); } else { - phase.Restart (); - var stagedFileData = await GetRemoteFileData (remoteStagingPath, runAs: false); - SetDiagnosticElapsed ("deploy.fastdeploy2.staging.stat.ms", phase); - if (stagedFileData == null) { + result = await UpdateOverrideCopies (remoteStagingPath, overridePath); + } + + if (result) { + WriteManifest (currentManifest); + await MarkRemoteReady (remoteStagingPath); + } + return result; + } + + bool UseShellSymlinkAppFileTransfer () + { + return string.Equals (AppFileTransferMode, "ShellSymlink", StringComparison.OrdinalIgnoreCase); + } + + async Task UpdateOverrideShellSymlinks (string remoteStagingPath, string overridePath, Dictionary currentManifest, Dictionary previousManifest, List removedFiles) + { + var newFiles = previousManifest == null ? + new HashSet (currentManifest.Keys, StringComparer.Ordinal) : + new HashSet (currentManifest.Keys.Where (file => !previousManifest.ContainsKey (file)), StringComparer.Ordinal); + SetDiagnosticProperty ("deploy.fastdeploy2.changed.files", newFiles.Count); + SetDiagnosticProperty ("deploy.symlink.created.files", newFiles.Count); + SetDiagnosticProperty ("deploy.symlink.removed.files", removedFiles.Count + newFiles.Count); + SetDiagnosticProperty ("deploy.fastdeploy2.stale.files", removedFiles.Count); + SetDiagnosticProperty ("deploy.symlink.tool.result", "shell"); + + var phase = Stopwatch.StartNew (); + if (!await RunCombinedShellSymlinkUpdate (remoteStagingPath, overridePath, currentManifest, previousManifest, newFiles, removedFiles)) { + SetDiagnosticElapsed ("deploy.symlink.shell.update.ms", phase); + return await FallbackToCopy (remoteStagingPath, overridePath); + } + SetDiagnosticElapsed ("deploy.symlink.shell.update.ms", phase); + + return true; + } + + async Task RunCombinedShellSymlinkUpdate (string remoteStagingPath, string overridePath, Dictionary currentManifest, Dictionary previousManifest, HashSet newFiles, List removedFiles) + { + var directories = new HashSet (StringComparer.Ordinal); + foreach (string file in currentManifest.Keys.Concat (removedFiles)) { + directories.Add (GetDirectoryName (file)); + } + + foreach (string directory in directories) { + var currentInDirectory = currentManifest.Keys.Where (file => GetDirectoryName (file) == directory).ToList (); + var newInDirectory = newFiles.Where (file => GetDirectoryName (file) == directory).ToList (); + var removedInDirectory = removedFiles.Where (file => GetDirectoryName (file) == directory).ToList (); + string targetDirectory = string.IsNullOrEmpty (directory) ? overridePath : $"{overridePath}/{directory}"; + string sourceDirectory = string.IsNullOrEmpty (directory) ? remoteStagingPath : $"{remoteStagingPath}/{directory}"; + + if (previousManifest == null || newInDirectory.Count == currentInDirectory.Count) { + string script = $"rm -f {ShellQuote (targetDirectory)}/*; mkdir -p {ShellQuote (targetDirectory)}; ln -sf {ShellQuote (sourceDirectory)}/* {ShellQuote (targetDirectory)}/"; + string output = await RunAs ("sh", "-c", script); + if (RaiseRunAsError (output) || IsShellError (output, "rm") || IsShellError (output, "mkdir") || IsShellError (output, "ln")) { + LogDiagnostic ($"Shell symlink glob update failed with '{output}'."); + return false; + } + continue; + } + + foreach (string script in CreateShellSymlinkScripts (remoteStagingPath, overridePath, newInDirectory, removedInDirectory)) { + string output = await RunAs ("sh", "-c", script); + if (RaiseRunAsError (output) || IsShellError (output, "rm") || IsShellError (output, "mkdir") || IsShellError (output, "ln")) { + LogDiagnostic ($"Shell symlink batch update failed with '{output}'."); + return false; + } + } + } + + return true; + } + + IEnumerable CreateShellSymlinkScripts (string remoteStagingPath, string overridePath, List newFiles, List removedFiles) + { + var filesToRemove = removedFiles.Concat (newFiles).Select (file => $"{overridePath}/{file}").ToList (); + foreach (var batch in BatchShellArguments ("rm -f", filesToRemove)) { + yield return batch; + } + + foreach (var group in newFiles.GroupBy (GetDirectoryName, StringComparer.Ordinal)) { + string targetDirectory = string.IsNullOrEmpty (group.Key) ? overridePath : $"{overridePath}/{group.Key}"; + var prefix = $"mkdir -p {ShellQuote (targetDirectory)}; ln -sf"; + var suffix = ShellQuote (targetDirectory) + "/"; + var sources = group.Select (file => $"{remoteStagingPath}/{file}"); + foreach (var batch in BatchShellArguments (prefix, sources, suffix)) { + yield return batch; + } + } + } + + IEnumerable BatchShellArguments (string prefix, IEnumerable arguments, string suffix = "") + { + var builder = new StringBuilder (prefix); + int count = 0; + foreach (string argument in arguments) { + string quoted = " " + ShellQuote (argument); + if (count > 0 && builder.Length + quoted.Length + suffix.Length >= MaxAdbCommandLength) { + if (!string.IsNullOrEmpty (suffix)) { + builder.Append (' ').Append (suffix); + } + yield return builder.ToString (); + builder.Clear (); + builder.Append (prefix); + count = 0; + } + builder.Append (quoted); + count++; + } + if (count > 0) { + if (!string.IsNullOrEmpty (suffix)) { + builder.Append (' ').Append (suffix); + } + yield return builder.ToString (); + } + } + + static string GetDirectoryName (string file) + { + return Path.GetDirectoryName (file)?.Replace ("\\", "/") ?? ""; + } + + static string ShellQuote (string value) + { + return "'" + value.Replace ("'", "'\"'\"'") + "'"; + } + + async Task RemoveOverridePaths (string overridePath, IEnumerable paths) + { + foreach (var batch in BatchArguments ("rm", "-f", paths.Select (file => $"{overridePath}/{file}"))) { + string output = await RunAs (batch.ToArray ()); + if (RaiseRunAsError (output) || IsShellError (output, "rm")) { + LogDiagnostic ($"Shell symlink remove failed with '{output}'."); return false; } + } + return true; + } + async Task CreateOverrideShellSymlinks (string remoteStagingPath, string overridePath, HashSet newFiles) + { + var filesByDirectory = new Dictionary> (StringComparer.Ordinal); + foreach (string file in newFiles) { + string directory = Path.GetDirectoryName (file)?.Replace ("\\", "/") ?? ""; + if (!filesByDirectory.TryGetValue (directory, out List files)) { + files = new List (); + filesByDirectory.Add (directory, files); + } + files.Add (file); + } + + var phase = Stopwatch.StartNew (); + foreach (var group in filesByDirectory) { + string targetDirectory = string.IsNullOrEmpty (group.Key) ? overridePath : $"{overridePath}/{group.Key}"; phase.Restart (); - var overrideFileData = await GetRemoteFileData (overridePath, runAs: true); - SetDiagnosticElapsed ("deploy.fastdeploy2.override.stat.ms", phase); - if (overrideFileData == null) { + string output = await RunAs ("mkdir", "-p", targetDirectory); + AddDiagnosticElapsed ("deploy.fastdeploy2.override.mkdir.ms", phase); + if (RaiseRunAsError (output) || IsShellError (output, "mkdir")) { + LogDiagnostic ($"Shell symlink mkdir failed with '{output}'."); return false; } - if (!await RemoveStaleOverrideFiles (overridePath, stagedFileData, overrideFileData)) { - return false; + for (int i = 0; i < group.Value.Count; i += 25) { + var args = new List { "ln", "-sf" }; + foreach (string file in group.Value.Skip (i).Take (25)) { + args.Add ($"{remoteStagingPath}/{file}"); + } + args.Add (targetDirectory); + phase.Restart (); + output = await RunAs (args.ToArray ()); + AddDiagnosticElapsed ("deploy.fastdeploy2.override.copy.ms", phase); + if (RaiseRunAsError (output) || IsShellError (output, "ln")) { + LogDiagnostic ($"Shell symlink ln failed with '{output}'."); + return false; + } } + } + + return true; + } + + async Task FallbackToCopy (string remoteStagingPath, string overridePath) + { + SetDiagnosticProperty ("deploy.symlink.tool.result", "shell fallback to copy"); + return await UpdateOverrideCopies (remoteStagingPath, overridePath); + } - result = await CopyChangedFiles (remoteStagingPath, overridePath, stagedFileData, overrideFileData); + async Task UpdateOverrideCopies (string remoteStagingPath, string overridePath) + { + var phase = Stopwatch.StartNew (); + var stagedFileData = await GetRemoteFileData (remoteStagingPath, runAs: false); + SetDiagnosticElapsed ("deploy.fastdeploy2.staging.stat.ms", phase); + if (stagedFileData == null) { + return false; } - if (result) { - WriteManifest (currentManifest); - await MarkRemoteReady (remoteStagingPath); + phase.Restart (); + var overrideFileData = await GetRemoteFileData (overridePath, runAs: true); + SetDiagnosticElapsed ("deploy.fastdeploy2.override.stat.ms", phase); + if (overrideFileData == null) { + return false; } - return result; + + if (!await RemoveStaleOverrideFiles (overridePath, stagedFileData, overrideFileData)) { + return false; + } + + return await CopyChangedFiles (remoteStagingPath, overridePath, stagedFileData, overrideFileData); } Dictionary CreateManifest (List files) diff --git a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets index ac07788a064..7ec9b0d357c 100644 --- a/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets +++ b/src/Xamarin.Android.Build.Debugging.Tasks/Xamarin.Android.Common.Debugging.targets @@ -333,8 +333,8 @@ Copyright (C) 2016 Xamarin. All rights reserved. Condition=" '$(_AndroidFastDevStrategy)' != 'FastDeploy' And '$(_AndroidFastDevStrategy)' != 'FastDeploy2' And '$(_AndroidFastDevStrategy)' != 'FastDeploy5' " Text="Invalid _AndroidFastDevStrategy value '$(_AndroidFastDevStrategy)'. Supported values are 'FastDeploy', 'FastDeploy2', and 'FastDeploy5'." /> + Condition=" '$(_AndroidFastDeployAppFileTransferMode)' != 'Copy' And '$(_AndroidFastDeployAppFileTransferMode)' != 'Symlink' And '$(_AndroidFastDeployAppFileTransferMode)' != 'ShellSymlink' " + Text="Invalid _AndroidFastDeployAppFileTransferMode value '$(_AndroidFastDeployAppFileTransferMode)'. Supported values are 'Copy', 'Symlink', and 'ShellSymlink'." />