[NEXUS-8301] Verify checksum while proxying

plugins/basic/nexus-repository-maven/src/main/java/org/sonatype/nexus/repository/maven/internal/policy/ChecksumPolicy.java → components/nexus-repository/src/main/java/org/sonatype/nexus/repository/proxy/ChecksumPolicy.java

@@ -10,10 +10,10 @@
  * of Sonatype, Inc. Apache Maven is a trademark of the Apache Software Foundation. M2eclipse is a trademark of the
  * Eclipse Foundation. All other trademarks are the property of their respective owners.
  */
-package org.sonatype.nexus.repository.maven.internal.policy;
+package org.sonatype.nexus.repository.proxy;
 
 /**
- * Maven checksum policy.
+ * Checksum policy.
  *
  * @since 3.0
  */

components/nexus-repository/src/main/java/org/sonatype/nexus/repository/proxy/ProxyFacetSupport.java

@@ -13,12 +13,18 @@
 package org.sonatype.nexus.repository.proxy;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.URI;
+import java.util.Map;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+import org.sonatype.nexus.common.collect.AttributesMap;
 import org.sonatype.nexus.common.collect.NestedAttributesMap;
+import org.sonatype.nexus.common.hash.HashAlgorithm;
+import org.sonatype.nexus.common.hash.Hashes;
+import org.sonatype.nexus.common.io.TempStreamSupplier;
 import org.sonatype.nexus.repository.FacetSupport;
 import org.sonatype.nexus.repository.MissingFacetException;
 import org.sonatype.nexus.repository.content.InvalidContentException;
@@ -28,8 +34,11 @@
 import org.sonatype.nexus.repository.view.Context;
 import org.sonatype.nexus.repository.view.Payload;
 import org.sonatype.nexus.repository.view.payloads.HttpEntityPayload;
+import org.sonatype.nexus.repository.view.payloads.StreamPayload;
+import org.sonatype.sisu.goodies.common.SimpleFormat;
 
 import com.google.common.base.Strings;
+import com.google.common.hash.HashCode;
 import com.google.common.net.HttpHeaders;
 import org.apache.http.Header;
 import org.apache.http.HttpEntity;
@@ -58,6 +67,8 @@ public abstract class ProxyFacetSupport
 
   private int artifactMaxAgeMinutes;
 
+  private ChecksumPolicy checksumPolicy;
+
   private HttpClientFacet httpClient;
 
   private boolean remoteUrlChanged;
@@ -80,6 +91,11 @@ protected void doConfigure() throws Exception {
 
     artifactMaxAgeMinutes = attributes.require("artifactMaxAge", Integer.class);
     log.debug("Artifact max age: {}", artifactMaxAgeMinutes);
+
+    checksumPolicy = ChecksumPolicy.valueOf(
+        attributes.get("checksumPolicy", String.class, ChecksumPolicy.IGNORE.name())
+    );
+    log.debug("Checksum policy: {}", checksumPolicy);
   }
 
 
@@ -124,7 +140,7 @@ public Content get(final Context context) throws IOException {
 
           // TODO: Introduce content validation.. perhaps content's type not matching path's implied type.
 
-          store(context, remote);
+          verifyAndStore(context, remote);
 
           content = getCachedPayload(context);
         }
@@ -137,6 +153,81 @@ public Content get(final Context context) throws IOException {
     return content;
   }
 
+  /**
+   * Verify (based on checksum policy) the fetched content against expected hashes and store it if verification passes.
+   */
+  private void verifyAndStore(final Context context, final Content content) throws IOException {
+    Map<HashAlgorithm, HashCode> expected;
+
+    if (checksumPolicy == ChecksumPolicy.IGNORE || (expected = getExpectedHashes(context)) == null) {
+      store(context, content);
+      return;
+    }
+
+    String message = null;
+    boolean stored = false;
+
+    if (expected.isEmpty()) {
+      message = SimpleFormat.format(
+          "No remote checksum available in repository %s for %s",
+          getRepository().getName(), getUrl(context)
+      );
+      if (checksumPolicy != ChecksumPolicy.STRICT) {
+        store(context, content);
+        stored = true;
+      }
+    }
+    else {
+      try (InputStream inputStream = content.openInputStream()) {
+        try (TempStreamSupplier supplier = new TempStreamSupplier(inputStream)) {
+          Map<HashAlgorithm, HashCode> actual;
+          try (InputStream is = supplier.get()) {
+            actual = Hashes.hash(expected.keySet(), is);
+          }
+          boolean hashesEquals = expected.equals(actual);
+          if (hashesEquals || checksumPolicy == ChecksumPolicy.WARN) {
+            store(context, new Content(new StreamPayload(supplier.get(), content.getSize(), content.getContentType()))
+            {
+              @Override
+              public AttributesMap getAttributes() {
+                return content.getAttributes();
+              }
+            });
+            stored = true;
+          }
+          if (!hashesEquals) {
+            message = SimpleFormat.format(
+                "Remote checksum of %s does not match in repository %s",
+                getUrl(context), getRepository().getName()
+            );
+          }
+        }
+      }
+    }
+    if (message != null) {
+      if (stored) {
+        log.warn(message);
+      }
+      else {
+        throw new InvalidContentException(message);
+      }
+    }
+  }
+
+  /**
+   * Get the expected hashes to be checked against the remotely retrieved asset (blob), check done based on configured
+   * checksum policy. If returned map is null, the check is skipped (as in repository format and/or asset does not
+   * support hashes). If returned map is empty, repository format and/or asset does support hashes but they are
+   * present,
+   * e.g. maven .sha1/.md5 are not present on remote)
+   *
+   * @param context request context
+   * @return expected hashes
+   */
+  protected Map<HashAlgorithm, HashCode> getExpectedHashes(final Context context) {
+    return null;
+  }
+
   /**
    * If we have the content cached locally already, return that - otherwise {@code null}.
    */
@@ -182,7 +273,8 @@ protected Content fetch(String url, Context context, Content stale) throws IOExc
 
       Payload payload = new HttpEntityPayload(response, entity);
       final Content result = new Content(payload);
-      result.getAttributes().set(Content.CONTENT_LAST_MODIFIED, extractLastModified(response.getLastHeader(HttpHeaders.LAST_MODIFIED)));
+      result.getAttributes()
+          .set(Content.CONTENT_LAST_MODIFIED, extractLastModified(response.getLastHeader(HttpHeaders.LAST_MODIFIED)));
       result.getAttributes().set(Content.CONTENT_ETAG, extractETag(response.getLastHeader(HttpHeaders.ETAG)));
       return result;
     }

plugins/basic/nexus-repository-maven/src/main/java/org/sonatype/nexus/repository/maven/internal/MavenDefaultRepositoriesContributor.groovy

@@ -20,7 +20,7 @@ import org.sonatype.nexus.repository.manager.DefaultRepositoriesContributor
 import org.sonatype.nexus.repository.maven.internal.maven2.Maven2GroupRecipe
 import org.sonatype.nexus.repository.maven.internal.maven2.Maven2HostedRecipe
 import org.sonatype.nexus.repository.maven.internal.maven2.Maven2ProxyRecipe
-import org.sonatype.nexus.repository.maven.internal.policy.ChecksumPolicy
+import org.sonatype.nexus.repository.proxy.ChecksumPolicy
 import org.sonatype.nexus.repository.maven.internal.policy.VersionPolicy
 import org.sonatype.nexus.repository.storage.WritePolicy
 
@@ -48,7 +48,6 @@ class MavenDefaultRepositoriesContributor
             [
                 maven  : [
                     versionPolicy              : VersionPolicy.RELEASE.toString(),
-                    checksumPolicy             : ChecksumPolicy.STRICT.toString(),
                     strictContentTypeValidation: false
                 ],
                 storage: [
@@ -61,7 +60,6 @@ class MavenDefaultRepositoriesContributor
             [
                 maven  : [
                     versionPolicy              : VersionPolicy.SNAPSHOT.toString(),
-                    checksumPolicy             : ChecksumPolicy.STRICT.toString(),
                     strictContentTypeValidation: false
                 ],
                 storage: [
@@ -73,12 +71,12 @@ class MavenDefaultRepositoriesContributor
             [
                 maven     : [
                     versionPolicy              : VersionPolicy.MIXED.toString(),
-                    checksumPolicy             : ChecksumPolicy.WARN.toString(),
                     strictContentTypeValidation: false
                 ],
                 proxy     : [
                     remoteUrl     : 'http://repo1.maven.org/maven2/',
-                    artifactMaxAge: 3600
+                    artifactMaxAge: 3600,
+                    checksumPolicy: ChecksumPolicy.WARN.toString(),
                 ],
                 httpclient: [
                     connection: [
@@ -95,7 +93,6 @@ class MavenDefaultRepositoriesContributor
             [
                 maven  : [
                     versionPolicy              : VersionPolicy.MIXED.toString(),
-                    checksumPolicy             : ChecksumPolicy.WARN.toString(),
                     strictContentTypeValidation: false
                 ],
                 storage: [

plugins/basic/nexus-repository-maven/src/main/java/org/sonatype/nexus/repository/maven/internal/MavenFacet.java

@@ -19,7 +19,6 @@
 
 import org.sonatype.nexus.repository.Facet;
 import org.sonatype.nexus.repository.content.InvalidContentException;
-import org.sonatype.nexus.repository.maven.internal.policy.ChecksumPolicy;
 import org.sonatype.nexus.repository.maven.internal.policy.VersionPolicy;
 import org.sonatype.nexus.repository.view.Content;
 import org.sonatype.nexus.repository.view.Payload;
@@ -47,12 +46,6 @@ public interface MavenFacet
   @Nonnull
   VersionPolicy getVersionPolicy();
 
-  /**
-   * Returns the checksum policy in effect for this repository.
-   */
-  @Nonnull
-  ChecksumPolicy getChecksumPolicy();
-
   @Nullable
   Content get(MavenPath path) throws IOException;
 

plugins/basic/nexus-repository-maven/src/main/java/org/sonatype/nexus/repository/maven/internal/MavenFacetImpl.java

@@ -34,7 +34,6 @@
 import org.sonatype.nexus.repository.content.InvalidContentException;
 import org.sonatype.nexus.repository.maven.internal.MavenPath.Coordinates;
 import org.sonatype.nexus.repository.maven.internal.MavenPath.HashType;
-import org.sonatype.nexus.repository.maven.internal.policy.ChecksumPolicy;
 import org.sonatype.nexus.repository.maven.internal.policy.VersionPolicy;
 import org.sonatype.nexus.repository.storage.Asset;
 import org.sonatype.nexus.repository.storage.Bucket;
@@ -119,8 +118,6 @@ public class MavenFacetImpl
 
   private VersionPolicy versionPolicy;
 
-  private ChecksumPolicy checksumPolicy;
-
   @Inject
   public MavenFacetImpl(final MimeSupport mimeSupport, final Map<String, MavenPathParser> mavenPathParsers) {
     this.mimeSupport = checkNotNull(mimeSupport);
@@ -137,9 +134,6 @@ protected void doConfigure() throws Exception {
     this.versionPolicy = VersionPolicy.valueOf(
         attributes.require("versionPolicy", String.class)
     );
-    this.checksumPolicy = ChecksumPolicy.valueOf(
-        attributes.require("checksumPolicy", String.class)
-    );
   }
 
   @Nonnull
@@ -154,12 +148,6 @@ public VersionPolicy getVersionPolicy() {
     return versionPolicy;
   }
 
-  @Nonnull
-  @Override
-  public ChecksumPolicy getChecksumPolicy() {
-    return checksumPolicy;
-  }
-
   @Nullable
   @Override
   public Content get(final MavenPath path) throws IOException {

plugins/basic/nexus-repository-maven/src/main/java/org/sonatype/nexus/repository/maven/internal/MavenProxyFacet.java

@@ -13,16 +13,28 @@
 package org.sonatype.nexus.repository.maven.internal;
 
 import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
 
 import javax.annotation.Nonnull;
 import javax.inject.Named;
 
+import org.sonatype.nexus.common.hash.HashAlgorithm;
 import org.sonatype.nexus.repository.content.InvalidContentException;
+import org.sonatype.nexus.repository.http.HttpMethods;
+import org.sonatype.nexus.repository.maven.internal.MavenPath.HashType;
 import org.sonatype.nexus.repository.proxy.ProxyFacetSupport;
 import org.sonatype.nexus.repository.view.Content;
 import org.sonatype.nexus.repository.view.Context;
+import org.sonatype.nexus.repository.view.Headers;
+import org.sonatype.nexus.repository.view.Parameters;
 import org.sonatype.nexus.repository.view.Payload;
+import org.sonatype.nexus.repository.view.PayloadResponse;
+import org.sonatype.nexus.repository.view.Request;
+import org.sonatype.nexus.repository.view.Response;
+import org.sonatype.nexus.repository.view.ViewFacet;
 
+import com.google.common.hash.HashCode;
 import org.joda.time.DateTime;
 
 /**
@@ -67,6 +79,53 @@ protected String getUrl(final @Nonnull Context context) {
     return context.getRequest().getPath().substring(1); // omit leading slash
   }
 
+  @Override
+  protected Map<HashAlgorithm, HashCode> getExpectedHashes(final Context context) {
+    MavenPath mavenPath = mavenPath(context);
+    if (mavenPath.isHash()) {
+      return null;
+    }
+    Map<HashAlgorithm, HashCode> hashes = new HashMap<>();
+    HashCode hash = getHash(mavenPath, HashType.SHA1);
+    if (hash != null) {
+      hashes.put(HashAlgorithm.SHA1, hash);
+    }
+    else {
+      hash = getHash(mavenPath, HashType.MD5);
+      if (hash != null) {
+        hashes.put(HashAlgorithm.MD5, hash);
+      }
+    }
+    return hashes;
+  }
+
+  private HashCode getHash(final MavenPath mavenPath, final HashType hashType) {
+    MavenPath hashMavenPath = mavenPath.hash(hashType);
+    // TODO use a request builder
+    Request request = new Request(hashMavenPath.getPath())
+    {
+      {
+        action = HttpMethods.GET;
+        parameters = new Parameters();
+        headers = new Headers();
+      }
+    };
+    try {
+      Response response = getRepository().facet(ViewFacet.class).dispatch(request);
+      if (response instanceof PayloadResponse) {
+        Payload payload = ((PayloadResponse) response).getPayload();
+        String hashCode = DigestExtractor.extract(payload.openInputStream());
+        if (hashCode != null) {
+          return HashCode.fromString(hashCode);
+        }
+      }
+    }
+    catch (Exception e) {
+      // ignore
+    }
+    return null;
+  }
+
   @Nonnull
   private MavenPath mavenPath(final @Nonnull Context context) {
     return context.getAttributes().require(MavenPath.class);

plugins/basic/nexus-repository-nuget/src/main/java/com/sonatype/nexus/repository/nuget/internal/NugetDefaultRepositoriesContributor.groovy

@@ -12,6 +12,8 @@
  */
 package com.sonatype.nexus.repository.nuget.internal
 
+import org.sonatype.nexus.repository.proxy.ChecksumPolicy
+
 import javax.inject.Named
 import javax.inject.Singleton
 
@@ -47,7 +49,8 @@ class NugetDefaultRepositoriesContributor
             [
                 proxy     : [
                     remoteUrl     : 'http://www.nuget.org/api/v2/',
-                    artifactMaxAge: 5
+                    artifactMaxAge: 5,
+                    checksumPolicy: ChecksumPolicy.WARN.toString()
                 ],
                 httpclient: [
                     connection: [