Add push file size limit and report func (and use it in githook) (#1173)

app/api/controller/githook/git.go

@@ -32,4 +32,8 @@ type RestrictedGIT interface {
  GetBranch(ctx context.Context, params *git.GetBranchParams) (*git.GetBranchOutput, error)
  Diff(ctx context.Context, in *git.DiffParams, files ...api.FileDiffRequest) (<-chan *git.FileDiff, <-chan error)
  GetBlob(ctx context.Context, params *git.GetBlobParams) (*git.GetBlobOutput, error)
+ FindOversizeFiles(
+ ctx context.Context,
+ params *git.FindOversizeFilesParams,
+ ) (*git.FindOversizeFilesOutput, error)
 }

app/api/controller/githook/pre_receive.go

@@ -93,6 +93,14 @@ func (c *Controller) PreReceive(
  return hook.Output{}, fmt.Errorf("failed to extend pre-receive hook: %w", err)
  }
 
+ err = c.checkFileSizeLimit(ctx, rgit, repo, in, &output)
+ if output.Error != nil {
+ return output, nil
+ }
+ if err != nil {
+ return hook.Output{}, err
+ }
+
  return output, nil
 }
 

app/api/controller/githook/pre_receive_file_size_limit.go

@@ -0,0 +1,81 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package githook
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/harness/gitness/app/services/settings"
+ "github.com/harness/gitness/git"
+ "github.com/harness/gitness/git/hook"
+ "github.com/harness/gitness/types"
+
+ "github.com/gotidy/ptr"
+)
+
+func (c *Controller) checkFileSizeLimit(
+ ctx context.Context,
+ rgit RestrictedGIT,
+ repo *types.Repository,
+ in types.GithookPreReceiveInput,
+ output *hook.Output,
+) error {
+ // return if all new refs are nil refs
+ allNilRefs := true
+ for _, refUpdate := range in.RefUpdates {
+ if refUpdate.New.IsNil() {
+ continue
+ }
+ allNilRefs = false
+ break
+ }
+ if allNilRefs {
+ return nil
+ }
+
+ sizeLimit, err := settings.RepoGet(
+ ctx,
+ c.settings,
+ repo.ID,
+ settings.KeyFileSizeLimit,
+ settings.DefaultFileSizeLimit,
+ )
+ if err != nil {
+ return fmt.Errorf("failed to check settings for file size limit: %w", err)
+ }
+ if sizeLimit <= 0 {
+ return nil
+ }
+
+ res, err := rgit.FindOversizeFiles(
+ ctx,
+ &git.FindOversizeFilesParams{
+ RepoUID: repo.GitUID,
+ GitObjectDirs: in.Environment.AlternateObjectDirs,
+ SizeLimit: sizeLimit,
+ },
+ )
+ if err != nil {
+ return fmt.Errorf("failed to get file sizes: %w", err)
+ }
+
+ if len(res.FileInfos) > 0 {
+ output.Error = ptr.String("Changes blocked by files exceeding the file size limit")
+ printOversizeFiles(output, res.FileInfos, sizeLimit)
+ }
+
+ return nil
+}

app/api/controller/githook/print.go

@@ -18,6 +18,7 @@ import (
  "fmt"
  "time"
 
+ "github.com/harness/gitness/git"
  "github.com/harness/gitness/git/hook"
 
  "github.com/fatih/color"
@@ -52,7 +53,7 @@ func printScanSecretsFindings(
  output.Messages,
  colorScanHeader.Sprintf(
  "Push contains %s:",
- stringSecretOrSecrets(findingsCnt > 1),
+ singularOrPlural("secret", findingsCnt > 1),
  ),
  "", // add empty line for making it visually more consumable
  )
@@ -81,19 +82,12 @@ func printScanSecretsFindings(
  colorScanSummary.Sprintf(
  "%d %s found",
  findingsCnt,
- stringSecretOrSecrets(findingsCnt > 1),
+ singularOrPlural("secret", findingsCnt > 1),
  )+fmt.Sprintf(" in %s", FMTDuration(time.Millisecond)),
  "", "", // add two empty lines for making it visually more consumable
  )
 }
 
-func stringSecretOrSecrets(plural bool) string {
- if plural {
- return "secrets"
- }
- return "secret"
-}
-
 func FMTDuration(d time.Duration) string {
  const secondsRounding = time.Second / time.Duration(10)
  switch {
@@ -108,3 +102,43 @@ func FMTDuration(d time.Duration) string {
  }
  return d.String()
 }
+
+func printOversizeFiles(
+ output *hook.Output,
+ oversizeFiles []git.FileInfo,
+ sizeLimit int64,
+) {
+ output.Messages = append(
+ output.Messages,
+ colorScanHeader.Sprintf(
+ "Push contains files exceeding the size limit:",
+ ),
+ "", // add empty line for making it visually more consumable
+ )
+
+ for _, file := range oversizeFiles {
+ output.Messages = append(
+ output.Messages,
+ fmt.Sprintf(" %s", file.SHA),
+ fmt.Sprintf(" Size: %dB", file.Size),
+ "", // add empty line for making it visually more consumable
+ )
+ }
+
+ total := len(oversizeFiles)
+ output.Messages = append(
+ output.Messages,
+ colorScanSummary.Sprintf(
+ "%d %s found exceeding the size limit of %dB",
+ total, singularOrPlural("file", total > 1), sizeLimit,
+ ),
+ "", "", // add two empty lines for making it visually more consumable
+ )
+}
+
+func singularOrPlural(noun string, plural bool) string {
+ if plural {
+ return noun + "s"
+ }
+ return noun
+}

app/api/controller/reposettings/general.go

@@ -0,0 +1,50 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reposettings
+
+import (
+ "github.com/harness/gitness/app/services/settings"
+
+ "github.com/gotidy/ptr"
+)
+
+// GeneralSettings represent the general repository settings as exposed externally.
+type GeneralSettings struct {
+ FileSizeLimit *int64 `json:"file_size_limit" yaml:"file_size_limit"`
+}
+
+func GetDefaultGeneralSettings() *GeneralSettings {
+ return &GeneralSettings{
+ FileSizeLimit: ptr.Int64(settings.DefaultFileSizeLimit),
+ }
+}
+
+func GetGeneralSettingsMappings(s *GeneralSettings) []settings.SettingHandler {
+ return []settings.SettingHandler{
+ settings.Mapping(settings.KeyFileSizeLimit, s.FileSizeLimit),
+ }
+}
+
+func GetGeneralSettingsAsKeyValues(s *GeneralSettings) []settings.KeyValue {
+ kvs := make([]settings.KeyValue, 0, 1)
+
+ if s.FileSizeLimit != nil {
+ kvs = append(kvs, settings.KeyValue{
+ Key: settings.KeyFileSizeLimit,
+ Value: s.FileSizeLimit,
+ })
+ }
+ return kvs
+}

app/api/controller/reposettings/general_find.go

@@ -0,0 +1,44 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reposettings
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/harness/gitness/app/auth"
+ "github.com/harness/gitness/types/enum"
+)
+
+// GeneralFind returns the general settings of a repo.
+func (c *Controller) GeneralFind(
+ ctx context.Context,
+ session *auth.Session,
+ repoRef string,
+) (*GeneralSettings, error) {
+ repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true)
+ if err != nil {
+ return nil, err
+ }
+
+ out := GetDefaultGeneralSettings()
+ mappings := GetGeneralSettingsMappings(out)
+ err = c.settings.RepoMap(ctx, repo.ID, mappings...)
+ if err != nil {
+ return nil, fmt.Errorf("failed to map settings: %w", err)
+ }
+
+ return out, nil
+}

app/api/controller/reposettings/general_update.go

@@ -0,0 +1,75 @@
+// Copyright 2023 Harness, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reposettings
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/harness/gitness/app/auth"
+ "github.com/harness/gitness/app/paths"
+ "github.com/harness/gitness/audit"
+ "github.com/harness/gitness/types/enum"
+
+ "github.com/rs/zerolog/log"
+)
+
+// GeneralUpdate updates the general settings of the repo.
+func (c *Controller) GeneralUpdate(
+ ctx context.Context,
+ session *auth.Session,
+ repoRef string,
+ in *GeneralSettings,
+) (*GeneralSettings, error) {
+ repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoEdit, false)
+ if err != nil {
+ return nil, err
+ }
+
+ // read old settings values
+ old := GetDefaultGeneralSettings()
+ oldMappings := GetGeneralSettingsMappings(old)
+ err = c.settings.RepoMap(ctx, repo.ID, oldMappings...)
+ if err != nil {
+ return nil, fmt.Errorf("failed to map settings (old): %w", err)
+ }
+
+ err = c.settings.RepoSetMany(ctx, repo.ID, GetGeneralSettingsAsKeyValues(in)...)
+ if err != nil {
+ return nil, fmt.Errorf("failed to set settings: %w", err)
+ }
+
+ // read all settings and return complete config
+ out := GetDefaultGeneralSettings()
+ mappings := GetGeneralSettingsMappings(out)
+ err = c.settings.RepoMap(ctx, repo.ID, mappings...)
+ if err != nil {
+ return nil, fmt.Errorf("failed to map settings: %w", err)
+ }
+
+ err = c.auditService.Log(ctx,
+ session.Principal,
+ audit.NewResource(audit.ResourceTypeRepositorySettings, repo.Identifier),
+ audit.ActionUpdated,
+ paths.Parent(repo.Path),
+ audit.WithOldObject(old),
+ audit.WithNewObject(out),
+ )
+ if err != nil {
+ log.Ctx(ctx).Warn().Msgf("failed to insert audit log for update repository settings operation: %s", err)
+ }
+
+ return out, nil
+}