From ea064d24f2771f10d9d1d22c10139a7ac73b4154 Mon Sep 17 00:00:00 2001
From: Christian Pointner <equinox@helsinki.at>
Date: Wed, 15 Aug 2018 13:13:00 +0200
Subject: [PATCH] also return source file hash on api call
---
importer/fetch.go | 42 ++++++++++++++++++++++++++++--------------
importer/types.go | 5 +++--
2 files changed, 31 insertions(+), 16 deletions(-)
diff --git a/importer/fetch.go b/importer/fetch.go
index 0e8bbf0..7592403 100644
--- a/importer/fetch.go
+++ b/importer/fetch.go
@@ -25,12 +25,13 @@
package importer
import (
- "crypto/sha256"
- "encoding/hex"
+ "encoding/base64"
"io"
"sync/atomic"
"time"
"unsafe"
+
+ "golang.org/x/crypto/blake2b"
)
type devNull uint64
@@ -81,6 +82,11 @@ func (pw *progressWriter) Write(p []byte) (n int, err error) {
return
}
+type copyResult struct {
+ err error
+ hash string
+}
+
func (job *Job) fetch() error {
job.Progress.set(StepFetching, 0)
@@ -100,31 +106,39 @@ func (job *Job) fetch() error {
return err
}
- done := make(chan error)
+ done := make(chan copyResult)
go func() {
- hash := sha256.New()
+ hash, err := blake2b.New256(nil)
+ if err != nil {
+ panic("created hash function failed: " + err.Error())
+ }
src := io.TeeReader(job.source.r, hash)
written, err := io.Copy(&progressWriter{job, 0, conv}, src)
- hashHex := hex.EncodeToString(hash.Sum(nil))
- job.im.dbgLog.Printf("fetch(): done copying %d bytes from source (SHA256: %s)", written, hashHex)
- job.im.store.UpdateFileSourceHash(job.Group, job.ID, "sha256:"+hashHex)
- done <- err
+ if err != nil {
+ done <- copyResult{err, ""}
+ return
+ }
+ hashStr := "blake2b_256:" + base64.URLEncoding.EncodeToString(hash.Sum(nil))
+ job.im.dbgLog.Printf("fetch(): done copying %d bytes from source (%s)", written, hashStr)
+ _, err = job.im.store.UpdateFileSourceHash(job.Group, job.ID, hashStr)
+ done <- copyResult{err, hashStr}
}()
+ var res copyResult
select {
case <-job.ctx.Done():
return job.ctx.Err()
- case err = <-done:
+ case res = <-done:
}
conv.Close()
- corr, convLog, convErr := conv.Wait()
- job.im.dbgLog.Printf("fetch(): converter returned: %f db, %v", corr, convErr)
- if convErr != nil {
- err = convErr
+ corr, convLog, err := conv.Wait()
+ job.im.dbgLog.Printf("fetch(): converter returned: %f db, %v", corr, err)
+ if err == nil {
+ err = res.err
}
job.Progress.set(StepFetching, 1)
- job.source.done <- &JobSourceResult{err, convLog}
+ job.source.done <- &JobSourceResult{err, res.hash, convLog}
return err
}
diff --git a/importer/types.go b/importer/types.go
index fff5700..b8c674d 100644
--- a/importer/types.go
+++ b/importer/types.go
@@ -75,8 +75,9 @@ func (s *SourceURL) UnmarshalText(data []byte) (err error) {
//******* Source
type JobSourceResult struct {
- Err error `json:"error,omitempty"`
- Log []string `json:"log"`
+ Err error `json:"error,omitempty"`
+ Hash string `json:"hash,omitempty"`
+ Log []string `json:"log"`
}
func (r *JobSourceResult) Error() string {
--
GitLab