/* * Copyright 2020 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.gradle.configurationcache.fingerprint import com.google.common.collect.Maps.newConcurrentMap import com.google.common.collect.Sets.newConcurrentHashSet import org.gradle.api.Describable import org.gradle.api.artifacts.ModuleVersionIdentifier import org.gradle.api.artifacts.component.ModuleComponentIdentifier import org.gradle.api.artifacts.component.ModuleComponentSelector import org.gradle.api.file.FileCollection import org.gradle.api.internal.artifacts.configurations.ConfigurationInternal import org.gradle.api.internal.artifacts.configurations.ProjectDependencyObservedListener import org.gradle.api.internal.artifacts.configurations.dynamicversion.Expiry import org.gradle.api.internal.artifacts.ivyservice.ivyresolve.ChangingValueDependencyResolutionListener import org.gradle.api.internal.artifacts.ivyservice.resolveengine.projectresult.ResolvedProjectConfiguration import org.gradle.api.internal.file.FileCollectionFactory import org.gradle.api.internal.file.FileCollectionInternal import org.gradle.api.internal.file.FileCollectionStructureVisitor import org.gradle.api.internal.file.FileTreeInternal import org.gradle.api.internal.file.collections.DirectoryFileTreeFactory import org.gradle.api.internal.file.collections.FileSystemMirroringFileTree import org.gradle.api.internal.project.ProjectState import org.gradle.api.internal.provider.ValueSourceProviderFactory import org.gradle.api.internal.provider.sources.EnvironmentVariableValueSource import org.gradle.api.internal.provider.sources.EnvironmentVariablesPrefixedByValueSource import org.gradle.api.internal.provider.sources.FileContentValueSource import org.gradle.api.internal.provider.sources.GradlePropertyValueSource import org.gradle.api.internal.provider.sources.SystemPropertiesPrefixedByValueSource import org.gradle.api.internal.provider.sources.SystemPropertyValueSource import org.gradle.api.internal.provider.sources.process.ProcessOutputValueSource import org.gradle.api.provider.ValueSourceParameters import org.gradle.api.tasks.util.PatternSet import org.gradle.configurationcache.CoupledProjectsListener import org.gradle.configurationcache.InputTrackingState import org.gradle.configurationcache.UndeclaredBuildInputListener import org.gradle.configurationcache.extensions.uncheckedCast import org.gradle.configurationcache.fingerprint.ConfigurationCacheFingerprint.InputFile import org.gradle.configurationcache.fingerprint.ConfigurationCacheFingerprint.ValueSource import org.gradle.configurationcache.problems.DocumentationSection import org.gradle.configurationcache.problems.PropertyProblem import org.gradle.configurationcache.problems.PropertyTrace import org.gradle.configurationcache.problems.StructuredMessage import org.gradle.configurationcache.serialization.DefaultWriteContext import org.gradle.configurationcache.services.ConfigurationCacheEnvironment import org.gradle.configurationcache.services.EnvironmentChangeTracker import org.gradle.groovy.scripts.ScriptSource import org.gradle.internal.concurrent.CompositeStoppable import org.gradle.internal.execution.TaskExecutionTracker import org.gradle.internal.execution.UnitOfWork import org.gradle.internal.execution.WorkInputListener import org.gradle.internal.execution.fingerprint.InputFingerprinter import org.gradle.internal.execution.fingerprint.InputFingerprinter.InputVisitor import org.gradle.internal.hash.HashCode import org.gradle.internal.resource.local.FileResourceListener import org.gradle.internal.scripts.ScriptExecutionListener import org.gradle.util.Path import java.io.File import java.util.EnumSet internal class ConfigurationCacheFingerprintWriter( private val host: Host, buildScopedContext: DefaultWriteContext, projectScopedContext: DefaultWriteContext, private val fileCollectionFactory: FileCollectionFactory, private val directoryFileTreeFactory: DirectoryFileTreeFactory, private val taskExecutionTracker: TaskExecutionTracker, private val environmentChangeTracker: EnvironmentChangeTracker, private val inputTrackingState: InputTrackingState, ) : ValueSourceProviderFactory.ValueListener, ValueSourceProviderFactory.ComputationListener, WorkInputListener, ScriptExecutionListener, UndeclaredBuildInputListener, ChangingValueDependencyResolutionListener, ProjectDependencyObservedListener, CoupledProjectsListener, FileResourceListener, ConfigurationCacheEnvironment.Listener { interface Host { val gradleUserHomeDir: File val allInitScripts: List val startParameterProperties: Map val buildStartTime: Long val cacheIntermediateModels: Boolean fun fingerprintOf(fileCollection: FileCollectionInternal): HashCode fun hashCodeOf(file: File): HashCode? fun displayNameOf(file: File): String fun reportInput(input: PropertyProblem) fun location(consumer: String?): PropertyTrace } private val buildScopedWriter = ScopedFingerprintWriter(buildScopedContext) private val buildScopedSink = BuildScopedSink(host, buildScopedWriter) private val projectScopedWriter = ScopedFingerprintWriter(projectScopedContext) private val sinksForProject = newConcurrentMap() private val projectForThread = ThreadLocal() private val projectDependencies = newConcurrentHashSet() private val undeclaredSystemProperties = newConcurrentHashSet() private val systemPropertiesPrefixedBy = newConcurrentHashSet() private val undeclaredEnvironmentVariables = newConcurrentHashSet() private val environmentVariablesPrefixedBy = newConcurrentHashSet() private val reportedFiles = newConcurrentHashSet() private val reportedValueSources = newConcurrentHashSet() private var closestChangingValue: ConfigurationCacheFingerprint.ChangingDependencyResolutionValue? = null init { val initScripts = host.allInitScripts buildScopedSink.initScripts(initScripts) buildScopedSink.write( ConfigurationCacheFingerprint.GradleEnvironment( host.gradleUserHomeDir, jvmFingerprint(), host.startParameterProperties ) ) } /** * Stops all writers. * * **MUST ALWAYS BE CALLED** */ fun close() { synchronized(this) { closestChangingValue?.let { buildScopedSink.write(it) } } CompositeStoppable.stoppable(buildScopedWriter, projectScopedWriter).stop() } override fun onDynamicVersionSelection(requested: ModuleComponentSelector, expiry: Expiry, versions: Set) { // Only consider repositories serving at least one version of the requested module. // This is meant to avoid repetitively expiring cache entries due to a 404 response for the requested module metadata // from one of the configured repositories. if (versions.isEmpty()) return val expireAt = host.buildStartTime + expiry.keepFor.toMillis() onChangingValue(ConfigurationCacheFingerprint.DynamicDependencyVersion(requested.displayName, expireAt)) } override fun onChangingModuleResolve(moduleId: ModuleComponentIdentifier, expiry: Expiry) { val expireAt = host.buildStartTime + expiry.keepFor.toMillis() onChangingValue(ConfigurationCacheFingerprint.ChangingModule(moduleId.displayName, expireAt)) } private fun onChangingValue(changingValue: ConfigurationCacheFingerprint.ChangingDependencyResolutionValue) { synchronized(this) { if (closestChangingValue == null || closestChangingValue!!.expireAt > changingValue.expireAt) { closestChangingValue = changingValue } } } private fun isInputTrackingDisabled() = !inputTrackingState.isEnabledForCurrentThread() private fun isExecutingTask() = taskExecutionTracker.currentTask.isPresent override fun fileObserved(file: File) { fileObserved(file, null) } override fun fileObserved(file: File, consumer: String?) { if (isInputTrackingDisabled()) { return } // Ignore consumer for now, only used by Gradle internals and so shouldn't appear in the report. captureFile(file) } override fun systemPropertyRead(key: String, value: Any?, consumer: String?) { if (isInputTrackingDisabled()) { return } addSystemPropertyToFingerprint(key, value, consumer) } private fun addSystemPropertyToFingerprint(key: String, value: Any?, consumer: String? = null) { if (isSystemPropertyMutated(key)) { // Mutated values of the system properties are not part of the fingerprint, as their value is // set at the configuration time. Everything that reads a mutated property value should be saved // as a fixed value. return } sink().systemPropertyRead(key, value) reportUniqueSystemPropertyInput(key, consumer) } override fun envVariableRead(key: String, value: String?, consumer: String?) { if (isInputTrackingDisabled()) { return } addEnvVariableToFingerprint(key, value, consumer) } private fun addEnvVariableToFingerprint(key: String, value: String?, consumer: String? = null) { sink().envVariableRead(key, value) reportUniqueEnvironmentVariableInput(key, consumer) } override fun fileOpened(file: File, consumer: String?) { if (isInputTrackingDisabled() || isExecutingTask()) { // Ignore files that are read as part of the task actions. These should really be task // inputs. Otherwise, we risk fingerprinting temporary files that will be gone at the // end of the build. return } captureFile(file) reportUniqueFileInput(file, consumer) } override fun fileCollectionObserved(fileCollection: FileCollection, consumer: String) { if (isInputTrackingDisabled()) { return } captureWorkInputs(consumer) { it(fileCollection as FileCollectionInternal) } } override fun systemPropertiesPrefixedBy(prefix: String, snapshot: Map) { if (isInputTrackingDisabled()) { return } addSystemPropertiesPrefixedByToFingerprint(prefix, snapshot) } private fun addSystemPropertiesPrefixedByToFingerprint(prefix: String, snapshot: Map) { val filteredSnapshot = snapshot.mapValues { e -> if (isSystemPropertyMutated(e.key)) { ConfigurationCacheFingerprint.SystemPropertiesPrefixedBy.IGNORED } else { e.value } } buildScopedSink.write(ConfigurationCacheFingerprint.SystemPropertiesPrefixedBy(prefix, filteredSnapshot)) } override fun envVariablesPrefixedBy(prefix: String, snapshot: Map) { if (isInputTrackingDisabled()) { return } addEnvVariablesPrefixedByToFingerprint(prefix, snapshot) } private fun addEnvVariablesPrefixedByToFingerprint(prefix: String, snapshot: Map) { buildScopedSink.write(ConfigurationCacheFingerprint.EnvironmentVariablesPrefixedBy(prefix, snapshot)) } override fun beforeValueObtained() { // Do not track additional inputs while computing a value of the value source. inputTrackingState.disableForCurrentThread() } override fun afterValueObtained() { inputTrackingState.restoreForCurrentThread() } override fun valueObtained( obtainedValue: ValueSourceProviderFactory.ValueListener.ObtainedValue, source: org.gradle.api.provider.ValueSource ) { // TODO(https://github.com/gradle/gradle/issues/22494) ValueSources become part of the fingerprint even if they are only obtained // inside other value sources. This is not really necessary for the correctness and causes excessive cache invalidation. when (val parameters = obtainedValue.valueSourceParameters) { is FileContentValueSource.Parameters -> { parameters.file.orNull?.asFile?.let { file -> // TODO - consider the potential race condition in computing the hash code here captureFile(file) reportUniqueFileInput(file) } } is GradlePropertyValueSource.Parameters -> { // The set of Gradle properties is already an input } is SystemPropertyValueSource.Parameters -> { addSystemPropertyToFingerprint(parameters.propertyName.get(), obtainedValue.value.get()) } is SystemPropertiesPrefixedByValueSource.Parameters -> { val prefix = parameters.prefix.get() addSystemPropertiesPrefixedByToFingerprint(prefix, obtainedValue.value.get().uncheckedCast()) reportUniqueSystemPropertiesPrefixedByInput(prefix) } is EnvironmentVariableValueSource.Parameters -> { addEnvVariableToFingerprint(parameters.variableName.get(), obtainedValue.value.get() as? String) } is EnvironmentVariablesPrefixedByValueSource.Parameters -> { val prefix = parameters.prefix.get() addEnvVariablesPrefixedByToFingerprint(prefix, obtainedValue.value.get().uncheckedCast()) reportUniqueEnvironmentVariablesPrefixedByInput(prefix) } is ProcessOutputValueSource.Parameters -> { sink().write(ValueSource(obtainedValue.uncheckedCast())) reportExternalProcessOutputRead(ProcessOutputValueSource.Parameters.getExecutable(parameters)) } else -> { sink().write(ValueSource(obtainedValue.uncheckedCast())) reportUniqueValueSourceInput( displayName = when (source) { is Describable -> source.displayName else -> null }, typeName = obtainedValue.valueSourceType.simpleName ) } } } private fun isSystemPropertyMutated(key: String): Boolean { return environmentChangeTracker.isSystemPropertyMutated(key) } override fun onScriptClassLoaded(source: ScriptSource, scriptClass: Class<*>) { source.resource.file?.let { captureFile(it) } } override fun onExecute(work: UnitOfWork, relevantTypes: EnumSet) { captureWorkInputs(work, relevantTypes) } private fun captureFile(file: File) { sink().captureFile(file) } private fun captureWorkInputs(work: UnitOfWork, relevantTypes: EnumSet) { captureWorkInputs(work.displayName) { visitStructure -> work.visitRegularInputs(object : InputVisitor { override fun visitInputFileProperty(propertyName: String, type: InputFingerprinter.InputPropertyType, value: InputFingerprinter.FileValueSupplier) { if (relevantTypes.contains(type)) { visitStructure(value.files as FileCollectionInternal) } } }) } } private inline fun captureWorkInputs(workDisplayName: String, content: ((FileCollectionInternal) -> Unit) -> Unit) { val fileSystemInputs = simplify(content) sink().write( ConfigurationCacheFingerprint.WorkInputs( workDisplayName, fileSystemInputs, host.fingerprintOf(fileSystemInputs) ) ) } private inline fun simplify(content: ((FileCollectionInternal) -> Unit) -> Unit): FileCollectionInternal { val simplifyingVisitor = SimplifyingFileCollectionStructureVisitor(directoryFileTreeFactory, fileCollectionFactory) content { it.visitStructure(simplifyingVisitor) } return simplifyingVisitor.simplify() } fun collectFingerprintForProject(identityPath: Path, action: () -> T): T { val previous = projectForThread.get() val projectSink = sinksForProject.computeIfAbsent(identityPath) { ProjectScopedSink(host, identityPath, projectScopedWriter) } projectForThread.set(projectSink) try { return action() } finally { projectForThread.set(previous) } } override fun dependencyObserved(consumingProject: ProjectState?, targetProject: ProjectState, requestedState: ConfigurationInternal.InternalState, target: ResolvedProjectConfiguration) { if (host.cacheIntermediateModels && consumingProject != null) { val dependency = ProjectSpecificFingerprint.ProjectDependency(consumingProject.identityPath, targetProject.identityPath) if (projectDependencies.add(dependency)) { projectScopedWriter.write(dependency) } } } override fun onProjectReference(referrer: ProjectState, target: ProjectState) { if (host.cacheIntermediateModels) { val dependency = ProjectSpecificFingerprint.CoupledProjects(referrer.identityPath, target.identityPath) if (projectDependencies.add(dependency)) { projectScopedWriter.write(dependency) } } } fun append(fingerprint: ProjectSpecificFingerprint) { // TODO - should add to report as an input projectScopedWriter.write(fingerprint) } private fun sink(): Sink = projectForThread.get() ?: buildScopedSink /** * Transform the collection into a sequence of files or directory trees and remove dynamic behaviour */ private class SimplifyingFileCollectionStructureVisitor( private val directoryFileTreeFactory: DirectoryFileTreeFactory, private val fileCollectionFactory: FileCollectionFactory ) : FileCollectionStructureVisitor { private val elements = mutableListOf() override fun visitCollection(source: FileCollectionInternal.Source, contents: Iterable) { elements.addAll(contents) } override fun visitGenericFileTree(fileTree: FileTreeInternal, sourceTree: FileSystemMirroringFileTree) { elements.addAll(fileTree) } override fun visitFileTree(root: File, patterns: PatternSet, fileTree: FileTreeInternal) { elements.add(directoryFileTreeFactory.create(root, patterns)) } override fun visitFileTreeBackedByFile(file: File, fileTree: FileTreeInternal, sourceTree: FileSystemMirroringFileTree) { elements.add(file) } fun simplify(): FileCollectionInternal = fileCollectionFactory.resolving(elements) } private fun reportUniqueValueSourceInput(displayName: String?, typeName: String) { // We assume different types won't ever produce identical display names if (reportedValueSources.add(displayName ?: typeName)) { reportValueSourceInput(displayName, typeName) } } private fun reportValueSourceInput(displayName: String?, typeName: String) { reportInput(consumer = null, documentationSection = null) { text("value from custom source ") reference(typeName) displayName?.let { text(", ") text(it) } } } private fun reportUniqueFileInput(file: File, consumer: String? = null) { if (reportedFiles.add(file)) { reportFileInput(file, consumer) } } private fun reportFileInput(file: File, consumer: String?) { reportInput(consumer, null) { text("file ") reference(host.displayNameOf(file)) } } private fun reportExternalProcessOutputRead(executable: String) { reportInput(consumer = null, documentationSection = DocumentationSection.RequirementsExternalProcess) { text("output of external process ") reference(executable) } } private fun reportUniqueSystemPropertyInput(key: String, consumer: String?) { if (undeclaredSystemProperties.add(key)) { reportSystemPropertyInput(key, consumer) } } private fun reportSystemPropertyInput(key: String, consumer: String?) { reportInput(consumer, DocumentationSection.RequirementsSysPropEnvVarRead) { text("system property ") reference(key) } } private fun reportUniqueSystemPropertiesPrefixedByInput(prefix: String) { if (systemPropertiesPrefixedBy.add(prefix)) { reportSystemPropertiesPrefixedByInput(prefix) } } private fun reportSystemPropertiesPrefixedByInput(prefix: String) { reportInput(null, DocumentationSection.RequirementsSysPropEnvVarRead) { if (prefix.isNotEmpty()) { text("system properties prefixed by ") reference(prefix) } else { text("system properties") } } } private fun reportUniqueEnvironmentVariableInput(key: String, consumer: String?) { if (undeclaredEnvironmentVariables.add(key)) { reportEnvironmentVariableInput(key, consumer) } } private fun reportEnvironmentVariableInput(key: String, consumer: String?) { reportInput(consumer, DocumentationSection.RequirementsSysPropEnvVarRead) { text("environment variable ") reference(key) } } private fun reportUniqueEnvironmentVariablesPrefixedByInput(prefix: String) { if (environmentVariablesPrefixedBy.add(prefix)) { reportEnvironmentVariablesPrefixedByInput(prefix) } } private fun reportEnvironmentVariablesPrefixedByInput(prefix: String) { reportInput(null, DocumentationSection.RequirementsSysPropEnvVarRead) { if (prefix.isNotEmpty()) { text("environment variables prefixed by ") reference(prefix) } else { text("environment variables") } } } private fun reportInput( consumer: String?, documentationSection: DocumentationSection?, messageBuilder: StructuredMessage.Builder.() -> Unit ) { host.reportInput( PropertyProblem( locationFor(consumer), StructuredMessage.build(messageBuilder), null, documentationSection = documentationSection ) ) } private fun locationFor(consumer: String?) = host.location(consumer) private abstract class Sink( private val host: Host ) { val capturedFiles: MutableSet = newConcurrentHashSet() private val undeclaredSystemProperties = newConcurrentHashSet() private val undeclaredEnvironmentVariables = newConcurrentHashSet() fun captureFile(file: File) { if (!capturedFiles.add(file)) { return } write(inputFile(file)) } fun systemPropertyRead(key: String, value: Any?) { if (undeclaredSystemProperties.add(key)) { write(ConfigurationCacheFingerprint.UndeclaredSystemProperty(key, value)) } } fun envVariableRead(key: String, value: String?) { if (undeclaredEnvironmentVariables.add(key)) { write(ConfigurationCacheFingerprint.UndeclaredEnvironmentVariable(key, value)) } } abstract fun write(value: ConfigurationCacheFingerprint) fun inputFile(file: File) = InputFile( file, host.hashCodeOf(file) ) } private class BuildScopedSink( host: Host, private val writer: ScopedFingerprintWriter ) : Sink(host) { override fun write(value: ConfigurationCacheFingerprint) { writer.write(value) } fun initScripts(initScripts: List) { capturedFiles.addAll(initScripts) write( ConfigurationCacheFingerprint.InitScripts( initScripts.map(::inputFile) ) ) } } private class ProjectScopedSink( host: Host, private val project: Path, private val writer: ScopedFingerprintWriter ) : Sink(host) { override fun write(value: ConfigurationCacheFingerprint) { writer.write(ProjectSpecificFingerprint.ProjectFingerprint(project, value)) } } } internal fun jvmFingerprint() = String.format( "%s|%s|%s", System.getProperty("java.vm.name"), System.getProperty("java.vm.vendor"), System.getProperty("java.vm.version") )