diff --git a/fields.go b/fields.go index f350f928..8a479729 100644 --- a/fields.go +++ b/fields.go @@ -52,7 +52,9 @@ func validateFieldsV0(publiccode PublicCode, parser Parser, network bool) error if _, err := isRelativePathOrURL(*publiccodev0.Logo, "logo"); err != nil { vr = append(vr, err) } else if !parser.disableExternalChecks { - validLogo, err := parser.validLogo(toCodeHostingURL(*publiccodev0.Logo, parser.currentBaseURL), network) + u, isGitRepo := toCodeHostingURL(*publiccodev0.Logo, parser.currentBaseURL, parser.allowLocalGitClone) + + validLogo, err := parser.validLogo(u, network, isGitRepo) if !validLogo { vr = append(vr, newValidationError("logo", err.Error())) } @@ -65,7 +67,9 @@ func validateFieldsV0(publiccode PublicCode, parser Parser, network bool) error if _, err := isRelativePathOrURL(*publiccodev0.MonochromeLogo, "monochromeLogo"); err != nil { vr = append(vr, err) } else if !parser.disableExternalChecks { - validLogo, err := parser.validLogo(toCodeHostingURL(*publiccodev0.MonochromeLogo, parser.currentBaseURL), network) + u, isGitRepo := toCodeHostingURL(*publiccodev0.MonochromeLogo, parser.currentBaseURL, parser.allowLocalGitClone) + + validLogo, err := parser.validLogo(u, network, isGitRepo) if !validLogo { vr = append(vr, newValidationError("monochromeLogo", err.Error())) } @@ -109,10 +113,10 @@ func validateFieldsV0(publiccode PublicCode, parser Parser, network bool) error if _, err := isRelativePathOrURL(*publiccodev0.Legal.AuthorsFile, "legal.authorsFile"); err != nil { vr = append(vr, err) } else if !parser.disableExternalChecks { - exists, err := parser.fileExists(toCodeHostingURL(*publiccodev0.Legal.AuthorsFile, parser.currentBaseURL), network) - if !exists { - u := toCodeHostingURL(*publiccodev0.Legal.AuthorsFile, parser.currentBaseURL) + u, isGitRepo := toCodeHostingURL(*publiccodev0.Legal.AuthorsFile, parser.currentBaseURL, parser.allowLocalGitClone) + exists, err := parser.fileExists(u, network, isGitRepo) + if !exists { vr = append(vr, newValidationError("legal.authorsFile", "'%s' does not exist: %s", urlutil.DisplayURL(&u), err.Error())) } } @@ -165,7 +169,9 @@ func validateFieldsV0(publiccode PublicCode, parser Parser, network bool) error if _, err := isRelativePathOrURL(v, keyName); err != nil { vr = append(vr, err) } else if !parser.disableExternalChecks { - isImage, err := parser.isImageFile(toCodeHostingURL(v, parser.currentBaseURL), network) + u, isGitRepo := toCodeHostingURL(v, parser.currentBaseURL, parser.allowLocalGitClone) + + isImage, err := parser.isImageFile(u, network, isGitRepo) if !isImage { vr = append(vr, newValidationError( keyName, diff --git a/go.mod b/go.mod index 0803bcc9..af4e96d5 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ require ( github.com/alranel/go-vcsurl/v2 v2.0.2 github.com/dyatlov/go-oembed v0.0.0-20191103150536-a57c85b3b37c github.com/github/go-spdx/v2 v2.3.3 + github.com/go-git/go-git/v5 v5.16.4 github.com/go-playground/locales v0.14.1 github.com/go-playground/universal-translator v0.18.1 github.com/go-playground/validator/v10 v10.18.0 @@ -13,15 +14,31 @@ require ( ) require ( + dario.cat/mergo v1.0.0 // indirect github.com/Jeffail/gabs v1.4.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/ProtonMail/go-crypto v1.1.6 // indirect + github.com/cloudflare/circl v1.6.1 // indirect + github.com/cyphar/filepath-securejoin v0.4.1 // indirect + github.com/emirpasic/gods v1.18.1 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.6.2 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/kevinburke/ssh_config v1.2.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/pjbgf/sha1cd v0.3.2 // indirect + github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/skeema/knownhosts v1.3.1 // indirect github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect - golang.org/x/crypto v0.36.0 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.23.0 // indirect + github.com/xanzy/ssh-agent v0.3.3 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect ) go 1.23.0 diff --git a/go.sum b/go.sum index a988d79e..dd032585 100644 --- a/go.sum +++ b/go.sum @@ -1,16 +1,45 @@ +dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/Jeffail/gabs v1.4.0 h1://5fYRRTq1edjfIrQGvdkcd22pkYUrHZ5YC/H2GJVAo= github.com/Jeffail/gabs v1.4.0/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= +github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/ProtonMail/go-crypto v1.1.6 h1:ZcV+Ropw6Qn0AX9brlQLAUXfqLBc7Bl+f/DmNxpLfdw= +github.com/ProtonMail/go-crypto v1.1.6/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE= github.com/alranel/go-vcsurl/v2 v2.0.2 h1:o8SRjpJEolgX/7eb2LY3WY1uEb+9sosKhW8uV3V3b7c= github.com/alranel/go-vcsurl/v2 v2.0.2/go.mod h1:pCJeg9/vFXM8bvHp45/Go98cRABcx5hKJgh9WZaYzqU= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0= +github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs= +github.com/cyphar/filepath-securejoin v0.4.1 h1:JyxxyPEaktOD+GAnqIqTf9A8tHyAG22rowi7HkoSU1s= +github.com/cyphar/filepath-securejoin v0.4.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dyatlov/go-oembed v0.0.0-20191103150536-a57c85b3b37c h1:MEV1LrQtCBGacXajlT4CSuYWbZuLl/qaZVqwoOmwAbU= github.com/dyatlov/go-oembed v0.0.0-20191103150536-a57c85b3b37c/go.mod h1:DjlDZiZGRRKbiJZmiEiiXozsBQAQzHmxwHKFeXifL2g= +github.com/elazarl/goproxy v1.7.2 h1:Y2o6urb7Eule09PjlhQRGNsqRfPmYI3KKQLFpCAV3+o= +github.com/elazarl/goproxy v1.7.2/go.mod h1:82vkLNir0ALaW14Rc399OTTjyNREgmdL2cVoIbS6XaE= +github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= +github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/github/go-spdx/v2 v2.3.3 h1:QI7evnHWEfWkT54eJwkoV/f3a0xD3gLlnVmT5wQG6LE= github.com/github/go-spdx/v2 v2.3.3/go.mod h1:2ZxKsOhvBp+OYBDlsGnUMcchLeo2mrpEBn2L1C+U3IQ= +github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c= +github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= +github.com/go-git/go-billy/v5 v5.6.2/go.mod h1:rcFC2rAsp/erv7CMz9GczHcuD0D32fWzH+MJAU+jaUU= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= +github.com/go-git/go-git/v5 v5.16.4 h1:7ajIEZHZJULcyJebDLo99bGgS0jRrOxzZG4uCk2Yb2Y= +github.com/go-git/go-git/v5 v5.16.4/go.mod h1:4Ge4alE/5gPs30F2H1esi2gPd69R0C39lolkucHBOp8= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -19,46 +48,87 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtPsPm2S9NAZ5nl9U= github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/italia/httpclient-lib-go v0.0.2 h1:4bJLywTVd7qHPdKxJXvvhlXp436JTC4KA6dLhIl5a6c= github.com/italia/httpclient-lib-go v0.0.2/go.mod h1:b0/D3ULsBw8X+zEl7j/kSZmiMlUdj+agppneOvSq6eA= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= +github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= +github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= +github.com/onsi/gomega v1.34.1/go.mod h1:kU1QgUvBDLXBJq618Xvm2LUX6rSAfRaFRTcdOeDLwwY= +github.com/pjbgf/sha1cd v0.3.2 h1:a9wb0bp1oC2TGwStyn0Umc/IGKQnEgF0vVaZ8QF8eo4= +github.com/pjbgf/sha1cd v0.3.2/go.mod h1:zQWigSxVmsHEZow5qaLtPYxpcKMMQpa09ixqBxuCS6A= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8= github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/skeema/knownhosts v1.3.1 h1:X2osQ+RAjK76shCbvhHHHVl3ZlgDm8apHEHFqRjnBY8= +github.com/skeema/knownhosts v1.3.1/go.mod h1:r7KTdC8l4uxWRyK2TpQZ/1o5HaSzh06ePQNxPwTcfiY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 h1:nrZ3ySNYwJbSpD6ce9duiP+QkD3JuLCcWkdaehUS/3Y= github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80/go.mod h1:iFyPdL66DjUD96XmzVL3ZntbzcflLnznH0fr99w5VqE= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM= +github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/git/git.go b/internal/git/git.go new file mode 100644 index 00000000..31aa43db --- /dev/null +++ b/internal/git/git.go @@ -0,0 +1,218 @@ +package git + +import ( + "fmt" + "net/url" + "os" + "path" + "path/filepath" + "strings" + + "github.com/go-git/go-git/v5" +) + +// Provides functionality for cloning and checking files in Git repositories. +type GitHelper struct { + // Base directory for all temporary Git clones + tempDir string + // Maps repository URLs to their go-git objects + repos map[string]*git.Repository + // Maps repository URLs to their local clone paths + ClonedRepos map[string]string +} + +func NewGitHelper() (*GitHelper, error) { + tempDir, err := os.MkdirTemp("", "publiccode-git-") + if err != nil { + return nil, fmt.Errorf("failed to create temp directory: %w", err) + } + + return &GitHelper{ + tempDir: tempDir, + repos: make(map[string]*git.Repository), + ClonedRepos: make(map[string]string), + }, nil +} + +// Performs a sparse clone of a Git repository. +func (g *GitHelper) CloneRepo(repoURL string) (string, error) { + // Check if already cloned + if clonePath, ok := g.ClonedRepos[repoURL]; ok { + return clonePath, nil + } + + // Create a repo name + repoName := strings.NewReplacer( + "http://", "", + "https://", "", + "/", "_", + ":", "_", + ).Replace(repoURL) + if len(repoName) > 100 { + repoName = repoName[:100] + } + + clonePath := filepath.Join(g.tempDir, repoName) + + repo, err := git.PlainClone(clonePath, false, &git.CloneOptions{ + URL: repoURL, + Depth: 1, + NoCheckout: true, + }) + if err != nil { + os.RemoveAll(clonePath) + + return "", fmt.Errorf("git clone failed: %w", err) + } + + g.repos[repoURL] = repo + g.ClonedRepos[repoURL] = clonePath + + return clonePath, nil +} + +// Checks out a specific file from the cloned repository. +func (g *GitHelper) CheckoutFile(repoPath string, filePath string) error { + // Find the repository for this path + var repo *git.Repository + for repoURL, path := range g.ClonedRepos { + if path == repoPath { + repo = g.repos[repoURL] + + break + } + } + + if repo == nil { + return fmt.Errorf("repository not found for path: %s", repoPath) + } + + w, err := repo.Worktree() + if err != nil { + return fmt.Errorf("git worktree failed: %w", err) + } + + // Use sparse checkout + dir := filepath.Dir(filePath) + if dir == "." { + dir = filePath + } + + err = w.Checkout(&git.CheckoutOptions{ + SparseCheckoutDirectories: []string{dir}, + }) + if err != nil { + return fmt.Errorf("git sparse-checkout failed: %w", err) + } + + // Check whether the file exists + fullPath := filepath.Join(repoPath, filePath) + if _, err := os.Stat(fullPath); err != nil { + return fmt.Errorf("git checkout failed: file not found: %s", filePath) + } + + return nil +} + +// Checks if a file exists in a Git repository by attempting to check it out. +func (g *GitHelper) FileExistsInRepo(repoURL string, filePath string) (bool, string, error) { + // Clone the repository if not already cloned + clonePath, err := g.CloneRepo(repoURL) + if err != nil { + return false, "", err + } + + // Try to checkout the file + err = g.CheckoutFile(clonePath, filePath) + if err != nil { + // File doesn't exist in the repository + return false, "", err + } + + // File exists and has been checked out + localPath := filepath.Join(clonePath, filePath) + if _, err := os.Stat(localPath); err != nil { + // File was supposedly checked out but doesn't exist - this is an error + return false, "", fmt.Errorf("file was checked out but not found at %s: %w", localPath, err) + } + + return true, localPath, nil +} + +// Removes all temporary directories and cloned repositories. +func (g *GitHelper) Cleanup() error { + if g.tempDir != "" { + return os.RemoveAll(g.tempDir) + } + + return nil +} + +// Checks if an URL is a generic Git repository URL. +// Returns false for supported hosting platforms (GitHub, GitLab, Bitbucket) +// which have web interfaces and should not use local Git cloning. +func IsGitURL(u *url.URL) bool { + if u == nil { + return false + } + + host := strings.ToLower(u.Host) + switch host { + case "github.com": + return false + case "gitlab.com": + return false + case "bitbucket.org": + return false + } + + if u.Scheme == "git" { + return true + } + + if u.Scheme == "http" || u.Scheme == "https" { + if strings.HasSuffix(u.Path, ".git") || + strings.Contains(u.Path, ".git/") { + return true + } + } + + return false +} + +// Extracts the base repository URL in a generic Git repository. +func GetRepoURL(u *url.URL) string { + repoURL := *u + + // For generic Git repos, remove the file path if present + if idx := strings.Index(repoURL.Path, ".git/"); idx != -1 { + // Keep everything up to and including .git + repoURL.Path = repoURL.Path[:idx+4] + + return repoURL.String() + } + + return repoURL.String() +} + +// Extracts the file path from a generic Git repository URL. +func ExtractFilePathFromURL(u *url.URL) (string, error) { + urlPath := u.Path + + var filePath string + + if strings.Contains(urlPath, ".git/") { + if idx := strings.Index(urlPath, ".git/"); idx != -1 { + filePath = urlPath[idx+5:] + } + } + + if filePath == "" { + return "", fmt.Errorf("could not extract file path from URL: %s", u.String()) + } + + // Clean up the file path + filePath = path.Clean(filePath) + + return filePath, nil +} diff --git a/parser.go b/parser.go index 3835d999..95498b3a 100644 --- a/parser.go +++ b/parser.go @@ -21,6 +21,7 @@ import ( "github.com/go-playground/validator/v10" en_translations "github.com/go-playground/validator/v10/translations/en" urlutil "github.com/italia/publiccode-parser-go/v5/internal" + "github.com/italia/publiccode-parser-go/v5/internal/git" publiccodeValidator "github.com/italia/publiccode-parser-go/v5/validators" "gopkg.in/yaml.v3" ) @@ -47,12 +48,18 @@ type ParserConfig struct { // The URL used as base of relative files in publiccode.yml (eg. logo) // It can be a local file with the 'file' scheme. BaseURL string + + // AllowLocalGitClone enables cloning Git repositories locally to check + // file existence for repositories not belonging to supported hosting providers + // (GitHub, GitLab and Bitbucket). + AllowLocalGitClone bool } // Parser is a helper class for parsing publiccode.yml files. type Parser struct { disableNetwork bool disableExternalChecks bool + allowLocalGitClone bool domain Domain branch string baseURL *url.URL @@ -64,6 +71,10 @@ type Parser struct { // XXX: It's an hack and it requires to fix the design mistake in the public API. // This makes Parse{,Stream}() not thread safe. currentBaseURL *url.URL + + // Holds temporarily cloned Git repositories + // for file existence checks during parsing session. + gitRepoCache map[string]string } // Domain is a single code hosting service. @@ -84,8 +95,10 @@ func NewParser(config ParserConfig) (*Parser, error) { parser := Parser{ disableNetwork: config.DisableNetwork, disableExternalChecks: config.DisableExternalChecks, + allowLocalGitClone: config.AllowLocalGitClone, domain: config.Domain, branch: config.Branch, + gitRepoCache: make(map[string]string), } if config.BaseURL != "" { @@ -102,6 +115,38 @@ func NewDefaultParser() (*Parser, error) { return NewParser(ParserConfig{}) } +// Cleanup removes any temporary Git repositories cloned during parsing. +func (p *Parser) Cleanup() error { + if p.gitRepoCache == nil { + return nil + } + + tempDirs := make(map[string]bool) + + // Collect all temp directories to clean up + for _, repoPath := range p.gitRepoCache { + if repoPath != "" { + // Get the parent temp directory + parent := filepath.Dir(repoPath) + if strings.Contains(parent, "publiccode-git-") { + tempDirs[parent] = true + } + } + } + + // Clean up all unique temp directories + for tempDir := range tempDirs { + if err := os.RemoveAll(tempDir); err != nil { + fmt.Fprintf(os.Stderr, "failed to remove temp Git directory %s: %v\n", tempDir, err) + } + } + + // Clear the cache + p.gitRepoCache = make(map[string]string) + + return nil +} + // ParseStream reads the data and tries to parse it. Returns an error if fails. func (p *Parser) ParseStream(in io.Reader) (PublicCode, error) { //nolint:maintidx b, err := io.ReadAll(in) @@ -360,6 +405,70 @@ func (p *Parser) Parse(uri string) (PublicCode, error) { return p.ParseStream(stream) } +// Checks if a file exists in a Git repository. +func (p *Parser) checkFileInGitRepo(u *url.URL) (bool, string, error) { + if !p.allowLocalGitClone { + return false, "", fmt.Errorf("local Git clone not allowed") + } + + // Extract repository URL and file path + repoURL := git.GetRepoURL(u) + + filePath, err := git.ExtractFilePathFromURL(u) + if err != nil { + return false, "", fmt.Errorf("failed to extract file path from URL %s: %w", u.String(), err) + } + + if p.gitRepoCache == nil { + p.gitRepoCache = make(map[string]string) + } + + if cachedPath, ok := p.gitRepoCache[repoURL]; ok { + // Check if file exists in cached repo + localPath := filepath.Join(cachedPath, filePath) + if _, err := os.Stat(localPath); err == nil { + return true, localPath, nil + } + } + + // Create a temporary Git helper + helper, err := git.NewGitHelper() + if err != nil { + return false, "", fmt.Errorf("failed to create Git helper: %w", err) + } + // Don't cleanup the temp directory, will be managed by the Parser + + // Check if file exists in repository + exists, localPath, err := helper.FileExistsInRepo(repoURL, filePath) + if err != nil { + // Clean up on error + if cleanupErr := helper.Cleanup(); cleanupErr != nil { + fmt.Fprintf(os.Stderr, "failed to cleanup Git helper: %v\n", cleanupErr) + } + + return false, "", fmt.Errorf("failed to check file in repo %s: %w", repoURL, err) + } + + if exists { + // Cache the cloned repo path for future use + // The path will be cleaned up when Parser.Cleanup() is called + if len(helper.ClonedRepos) > 0 { + for _, clonePath := range helper.ClonedRepos { + p.gitRepoCache[repoURL] = clonePath + + break + } + } + } else { + // If file doesn't exist, clean up immediately + if cleanupErr := helper.Cleanup(); cleanupErr != nil { + fmt.Fprintf(os.Stderr, "failed to cleanup Git helper: %v\n", cleanupErr) + } + } + + return exists, localPath, nil +} + // Ensure the returned value implements PublicCode as a struct, not as a pointer func asPublicCode(pc PublicCode) PublicCode { switch v := pc.(type) { diff --git a/publiccode-parser/publiccode_parser.go b/publiccode-parser/publiccode_parser.go index ab023bf6..f977e247 100644 --- a/publiccode-parser/publiccode_parser.go +++ b/publiccode-parser/publiccode_parser.go @@ -38,6 +38,7 @@ func main() { localBasePathPtr := flag.String("path", "", "Use this local directory as base path when checking for files existence instead of using the `url` key in publiccode.yml") disableNetworkPtr := flag.Bool("no-network", false, "Disables checks that require network connections (URL existence and oEmbed). This makes validation much faster.") disableExternalChecksPtr := flag.Bool("no-external-checks", false, "Disables ALL checks that reference external resources such as remote URLs or local file existence. Implies --no-network") + allowLocalGitClonePtr := flag.Bool("allow-local-git-clone", false, "Allow cloning Git repositories locally to check file existence for generic Git repos not supported by specific hosting providers.") jsonOutputPtr := flag.Bool("json", false, "Output the validation errors as a JSON list.") helpPtr := flag.Bool("help", false, "Display command line usage.") versionPtr := flag.Bool("version", false, "Display current software version.") @@ -68,6 +69,7 @@ func main() { config := publiccode.ParserConfig{BaseURL: *localBasePathPtr} config.DisableNetwork = *disableNetworkPtr config.DisableExternalChecks = *disableExternalChecksPtr + config.AllowLocalGitClone = *allowLocalGitClonePtr p, err := publiccode.NewParser(config) if err != nil { @@ -75,6 +77,15 @@ func main() { os.Exit(1) } + // Ensure cleanup of any temporary Git repositories + if config.AllowLocalGitClone { + defer func() { + if err := p.Cleanup(); err != nil { + fmt.Fprintf(os.Stderr, "failed to cleanup temporary Git repositories: %v\n", err) + } + }() + } + _, err = p.Parse(publiccodeFile) if *jsonOutputPtr { diff --git a/validations.go b/validations.go index cdc1e413..36316a05 100644 --- a/validations.go +++ b/validations.go @@ -19,6 +19,7 @@ import ( "github.com/dyatlov/go-oembed/oembed" httpclient "github.com/italia/httpclient-lib-go" "github.com/italia/publiccode-parser-go/v5/data" + "github.com/italia/publiccode-parser-go/v5/internal/git" netutil "github.com/italia/publiccode-parser-go/v5/internal" ) @@ -95,15 +96,22 @@ func (p *Parser) isReachable(u url.URL, network bool) (bool, error) { // code hosting URLs to their raw URL. // // It supports relative paths and turns them into remote URLs or file:// URLs -// depending on the value of baseURL -func toCodeHostingURL(file string, baseURL *url.URL) url.URL { +// depending on the value of baseURL. +// If a URL is not supported by vcsurl, but it looks like a Git repository, +// it returns the URL as-is. +func toCodeHostingURL(file string, baseURL *url.URL, allowGitClone bool) (url.URL, bool) { // Check if file is an absolute URL if uri, err := url.ParseRequestURI(file); err == nil { if raw := vcsurl.GetRawFile(uri); raw != nil { - return *raw + return *raw, false } - return *uri + // Check if it's a Git repository URL that we can clone + if allowGitClone && git.IsGitURL(uri) { + return *uri, true + } + + return *uri, false } // We always pass the computed base URL here, with a fallback to the cwd, @@ -112,11 +120,16 @@ func toCodeHostingURL(file string, baseURL *url.URL) url.URL { u := *baseURL u.Path = path.Join(u.Path, file) - return u + // Check if the base URL itself it's a Git repository + if allowGitClone && git.IsGitURL(&u) { + return u, true + } + + return u, false } -// fileExists returns true if the file resource exists. -func (p *Parser) fileExists(u url.URL, network bool) (bool, error) { +// Returns true if the file resource exists. +func (p *Parser) fileExists(u url.URL, network bool, isGitRepo bool) (bool, error) { // Don't check if we are running in WASM because there's no stat(2) there if runtime.GOARCH == "wasm" { return true, nil @@ -133,6 +146,15 @@ func (p *Parser) fileExists(u url.URL, network bool) (bool, error) { return err == nil, err } + // Check file existence via Git clone + if isGitRepo && p.allowLocalGitClone && network { + exists, _, err := p.checkFileInGitRepo(&u) + if err == nil { + return exists, nil + } + } + + // Check file existence via HTTP request if network { reachable, err := p.isReachable(u, network) @@ -142,9 +164,9 @@ func (p *Parser) fileExists(u url.URL, network bool) (bool, error) { return true, nil } -// isImageFile check whether the string is a valid image. It also checks if the file exists. -// It returns true if it is an image or false if it's not and an error, if any -func (p *Parser) isImageFile(u url.URL, network bool) (bool, error) { +// Checks whether the string is a valid image. It also checks if the file exists. +// It returns true if it is an image or false if it's not and an error, if any. +func (p *Parser) isImageFile(u url.URL, network bool, isGitRepo bool) (bool, error) { validExt := []string{".jpg", ".png"} ext := strings.ToLower(filepath.Ext(u.Path)) @@ -152,12 +174,12 @@ func (p *Parser) isImageFile(u url.URL, network bool) (bool, error) { return false, fmt.Errorf("invalid file extension for: %s", netutil.DisplayURL(&u)) } - return p.fileExists(u, network) + return p.fileExists(u, network, isGitRepo) } -// validLogo returns true if the file path in value is a valid logo. +// Returns true if the file path in value is a valid logo. // It also checks if the file exists. -func (p *Parser) validLogo(u url.URL, network bool) (bool, error) { +func (p *Parser) validLogo(u url.URL, network bool, isGitRepo bool) (bool, error) { validExt := []string{".svg", ".svgz", ".png"} ext := strings.ToLower(filepath.Ext(u.Path)) @@ -166,13 +188,22 @@ func (p *Parser) validLogo(u url.URL, network bool) (bool, error) { return false, fmt.Errorf("invalid file extension for: %s", netutil.DisplayURL(&u)) } - if exists, err := p.fileExists(u, network); !exists { + if exists, err := p.fileExists(u, network, isGitRepo); !exists { return false, err } var localPath string - // Remote. Create a temp dir, download and check the file. Remove the temp dir. - if u.Scheme != "file" { + + // If it's a Git repository, try to get the file locally + if isGitRepo && p.allowLocalGitClone && network { + exists, gitLocalPath, err := p.checkFileInGitRepo(&u) + if err == nil && exists { + localPath = gitLocalPath + } + } + + // If we don't have a local path yet and it's a remote file, download it + if localPath == "" && u.Scheme != "file" { var err error if !network { @@ -198,11 +229,11 @@ func (p *Parser) validLogo(u url.URL, network bool) (bool, error) { fmt.Fprintf(os.Stderr, "failed to remove %s: %v\n", dir, err) } }() - } else { + } else if localPath == "" && u.Scheme == "file" { localPath = u.Path } - if ext == ".png" { + if localPath != "" && ext == ".png" { image.RegisterFormat("png", "png", png.Decode, png.DecodeConfig) f, err := os.Open(localPath)