From fe017d99c6300d46e4a0735b0ba7214ca668b47c Mon Sep 17 00:00:00 2001 From: OlegChuev Date: Tue, 22 Apr 2025 17:13:03 +0300 Subject: [PATCH 1/7] feat(archive): add concurrency control for archive operations Introduce a new `archiveWorkers` channel to limit the number of concurrent archive operations. When the limit is reached, new requests will receive a 429 Too Many Requests response. This prevents resource exhaustion and improves system stability. The maximum number of workers can be configured via the `--max-archive-workers` CLI option. --- README.md | 9 ++++-- src/param/cli.go | 14 +++++--- src/param/main.go | 21 ++++++++---- src/serverHandler/aliasHandler.go | 53 ++++++++++++++++++++----------- 4 files changed, 64 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index ccca502..b31d99a 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Will generate executable file "main" in current directory. Start server on port 8080, root directory is current working directory: ```sh ghfs -l 8080 -``` +``` Start server on port 8080, root directory is /usr/share/doc: ```sh @@ -258,7 +258,10 @@ ghfs [options] -A|--global-archive Allow user to download the whole contents of current directory for all url paths. A download link will appear on top part of the page. - Make sure there is no circular symbol links. +--max-archive-workers + Maximum number of concurrent archive operations. + Set to -1 for unlimited (default). + When the limit is reached, new archive requests will receive 429 Too Many Requests. --archive ... --archive-user [...] ... Allow user to download the whole contents of current directory for specific url paths(and sub paths). @@ -304,7 +307,7 @@ ghfs [options] -S|--show ... -SD|--show-dir ... -SF|--show-file ... - If specified, files or directories match wildcards(except hidden by hide option) will be shown. + If specified, files or directories match wildcards(except hidden by hide option) will be shown. -H|--hide ... -HD|--hide-dir ... diff --git a/src/param/cli.go b/src/param/cli.go index e35a722..9574dc9 100644 --- a/src/param/cli.go +++ b/src/param/cli.go @@ -2,12 +2,13 @@ package param import ( "errors" - "mjpclab.dev/ghfs/src/goNixArgParser" - "mjpclab.dev/ghfs/src/goVirtualHost" - "mjpclab.dev/ghfs/src/serverError" "net/http" "os" "strings" + + "mjpclab.dev/ghfs/src/goNixArgParser" + "mjpclab.dev/ghfs/src/goVirtualHost" + "mjpclab.dev/ghfs/src/serverError" ) var cliCmd = NewCliCmd() @@ -147,6 +148,9 @@ func NewCliCmd() *goNixArgParser.Command { err = options.AddFlagValues("archivedirsusers", "--archive-dir-user", "", nil, "file system path that allow archive files for specific users, [...]") serverError.CheckFatal(err) + err = options.AddFlagValue("maxarchiveworkers", "--max-archive-workers", "", "-1", "maximum number of concurrent archive operations (-1 for unlimited)") + serverError.CheckFatal(err) + err = options.AddFlag("globalcors", "--global-cors", "GHFS_GLOBAL_CORS", "enable CORS headers for all directories") serverError.CheckFatal(err) @@ -436,6 +440,8 @@ func CmdResultsToParams(results []*goNixArgParser.ParseResult) (params Params, e archiveDirsUsers, _ := result.GetStrings("archivedirsusers") param.ArchiveDirsUsers = SplitAllKeyValues(archiveDirsUsers) + param.ArchiveMaxWorkers, _ = result.GetInt("maxarchiveworkers") + // global restrict access if result.HasKey("globalrestrictaccess") { param.GlobalRestrictAccess, _ = result.GetStrings("globalrestrictaccess") @@ -464,7 +470,7 @@ func CmdResultsToParams(results []*goNixArgParser.ParseResult) (params Params, e // certificate certFiles, _ := result.GetStrings("certs") keyFiles, _ := result.GetStrings("keys") - param.CertKeyPaths, es = goVirtualHost.CertsKeysToPairs(certFiles, keyFiles) + param.CertKeyPaths, _ = goVirtualHost.CertsKeysToPairs(certFiles, keyFiles) // listen listens, _ := result.GetStrings("listens") diff --git a/src/param/main.go b/src/param/main.go index 170bb60..93f7f44 100644 --- a/src/param/main.go +++ b/src/param/main.go @@ -1,11 +1,12 @@ package param import ( + "os" + "path/filepath" + "mjpclab.dev/ghfs/src/middleware" "mjpclab.dev/ghfs/src/serverError" "mjpclab.dev/ghfs/src/util" - "os" - "path/filepath" ) type Param struct { @@ -56,11 +57,13 @@ type Param struct { DeleteDirs []string DeleteDirsUsers [][]string // [][path, user...] - GlobalArchive bool - ArchiveUrls []string - ArchiveUrlsUsers [][]string // [][path, user...] - ArchiveDirs []string - ArchiveDirsUsers [][]string // [][path, user...] + GlobalArchive bool + ArchiveUrls []string + ArchiveUrlsUsers [][]string // [][path, user...] + ArchiveDirs []string + ArchiveDirsUsers [][]string // [][path, user...] + ArchiveMaxWorkers int + ArchivationsSem chan struct{} GlobalCors bool CorsUrls []string @@ -163,6 +166,10 @@ func (param *Param) Normalize() (errs []error) { param.DeleteDirs = NormalizeFsPaths(param.DeleteDirs) param.ArchiveUrls = NormalizeUrlPaths(param.ArchiveUrls) param.ArchiveDirs = NormalizeFsPaths(param.ArchiveDirs) + if param.ArchiveMaxWorkers > 0 { + param.ArchivationsSem = make(chan struct{}, param.ArchiveMaxWorkers) + } + param.CorsUrls = NormalizeUrlPaths(param.CorsUrls) param.CorsDirs = NormalizeFsPaths(param.CorsDirs) diff --git a/src/serverHandler/aliasHandler.go b/src/serverHandler/aliasHandler.go index 21a3d7f..554e037 100644 --- a/src/serverHandler/aliasHandler.go +++ b/src/serverHandler/aliasHandler.go @@ -1,16 +1,17 @@ package serverHandler import ( + "net/http" + "regexp" + "strconv" + "strings" + "mjpclab.dev/ghfs/src/middleware" "mjpclab.dev/ghfs/src/param" "mjpclab.dev/ghfs/src/serverLog" "mjpclab.dev/ghfs/src/tpl/theme" "mjpclab.dev/ghfs/src/user" "mjpclab.dev/ghfs/src/util" - "net/http" - "regexp" - "strconv" - "strings" ) var defaultHandler = http.NotFoundHandler() @@ -48,6 +49,8 @@ type aliasHandler struct { archive *hierarchyAvailability cors *hierarchyAvailability + archiveWorkers chan struct{} + globalRestrictAccess []string restrictAccessUrls pathStringsList restrictAccessDirs pathStringsList @@ -120,21 +123,8 @@ func (h *aliasHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if session.isMutate && h.mutate(w, r, session, data) { return - } else if session.isArchive { - switch session.archiveFormat { - case tarFmt: - if h.tar(w, r, session, data) { - return - } - case tgzFmt: - if h.tgz(w, r, session, data) { - return - } - case zipFmt: - if h.zip(w, r, session, data) { - return - } - } + } else if session.isArchive && h.createArchive(w, r, session, data) { + return } } @@ -152,6 +142,29 @@ func (h *aliasHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } +func (h *aliasHandler) createArchive(w http.ResponseWriter, r *http.Request, session *sessionContext, data *responseData) bool { + if h.archiveWorkers != nil { + select { + case h.archiveWorkers <- struct{}{}: + defer func() { <-h.archiveWorkers }() + default: + data.Status = http.StatusTooManyRequests + return false + } + } + + switch session.archiveFormat { + case tarFmt: + return h.tar(w, r, session, data) + case tgzFmt: + return h.tgz(w, r, session, data) + case zipFmt: + return h.zip(w, r, session, data) + } + + return false +} + func newAliasHandler( p *param.Param, vhostCtx *vhostContext, @@ -179,6 +192,8 @@ func newAliasHandler( toHttpsPort: p.ToHttpsPort, defaultSort: p.DefaultSort, + archiveWorkers: p.ArchivationsSem, + users: vhostCtx.users, theme: vhostCtx.theme, logger: vhostCtx.logger, From 8c8f83d8178d212a0e8649d952dd45751b9c3b8c Mon Sep 17 00:00:00 2001 From: OlegChuev Date: Thu, 24 Apr 2025 22:10:30 +0300 Subject: [PATCH 2/7] refactor(archive): renamed key for max archive workers --- README.md | 2 +- src/param/cli.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b31d99a..1eedf37 100644 --- a/README.md +++ b/README.md @@ -258,7 +258,7 @@ ghfs [options] -A|--global-archive Allow user to download the whole contents of current directory for all url paths. A download link will appear on top part of the page. ---max-archive-workers +--archive-max-workers Maximum number of concurrent archive operations. Set to -1 for unlimited (default). When the limit is reached, new archive requests will receive 429 Too Many Requests. diff --git a/src/param/cli.go b/src/param/cli.go index 9574dc9..dd6ed0b 100644 --- a/src/param/cli.go +++ b/src/param/cli.go @@ -148,7 +148,7 @@ func NewCliCmd() *goNixArgParser.Command { err = options.AddFlagValues("archivedirsusers", "--archive-dir-user", "", nil, "file system path that allow archive files for specific users, [...]") serverError.CheckFatal(err) - err = options.AddFlagValue("maxarchiveworkers", "--max-archive-workers", "", "-1", "maximum number of concurrent archive operations (-1 for unlimited)") + err = options.AddFlagValue("archivemaxworkers", "--archive-max-workers", "", "-1", "maximum number of concurrent archive operations (-1 for unlimited)") serverError.CheckFatal(err) err = options.AddFlag("globalcors", "--global-cors", "GHFS_GLOBAL_CORS", "enable CORS headers for all directories") @@ -440,7 +440,7 @@ func CmdResultsToParams(results []*goNixArgParser.ParseResult) (params Params, e archiveDirsUsers, _ := result.GetStrings("archivedirsusers") param.ArchiveDirsUsers = SplitAllKeyValues(archiveDirsUsers) - param.ArchiveMaxWorkers, _ = result.GetInt("maxarchiveworkers") + param.ArchiveMaxWorkers, _ = result.GetInt("archivemaxworkers") // global restrict access if result.HasKey("globalrestrictaccess") { From fb74e3e73576c6631d235a24bbf56891a8dacef9 Mon Sep 17 00:00:00 2001 From: OlegChuev Date: Fri, 25 Apr 2025 09:43:39 +0300 Subject: [PATCH 3/7] refactor(archive): replaced channels with atomic int --- src/param/cli.go | 2 +- src/param/main.go | 6 +----- src/serverHandler/aliasHandler.go | 17 ++++++++++------- src/serverHandler/vhostHandler.go | 16 ++++++++++++++-- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/param/cli.go b/src/param/cli.go index dd6ed0b..8334296 100644 --- a/src/param/cli.go +++ b/src/param/cli.go @@ -440,7 +440,7 @@ func CmdResultsToParams(results []*goNixArgParser.ParseResult) (params Params, e archiveDirsUsers, _ := result.GetStrings("archivedirsusers") param.ArchiveDirsUsers = SplitAllKeyValues(archiveDirsUsers) - param.ArchiveMaxWorkers, _ = result.GetInt("archivemaxworkers") + param.ArchiveMaxWorkers, _ = result.GetInt32("archivemaxworkers") // global restrict access if result.HasKey("globalrestrictaccess") { diff --git a/src/param/main.go b/src/param/main.go index 93f7f44..3a401c0 100644 --- a/src/param/main.go +++ b/src/param/main.go @@ -62,8 +62,7 @@ type Param struct { ArchiveUrlsUsers [][]string // [][path, user...] ArchiveDirs []string ArchiveDirsUsers [][]string // [][path, user...] - ArchiveMaxWorkers int - ArchivationsSem chan struct{} + ArchiveMaxWorkers int32 GlobalCors bool CorsUrls []string @@ -166,9 +165,6 @@ func (param *Param) Normalize() (errs []error) { param.DeleteDirs = NormalizeFsPaths(param.DeleteDirs) param.ArchiveUrls = NormalizeUrlPaths(param.ArchiveUrls) param.ArchiveDirs = NormalizeFsPaths(param.ArchiveDirs) - if param.ArchiveMaxWorkers > 0 { - param.ArchivationsSem = make(chan struct{}, param.ArchiveMaxWorkers) - } param.CorsUrls = NormalizeUrlPaths(param.CorsUrls) param.CorsDirs = NormalizeFsPaths(param.CorsDirs) diff --git a/src/serverHandler/aliasHandler.go b/src/serverHandler/aliasHandler.go index 554e037..95f4aba 100644 --- a/src/serverHandler/aliasHandler.go +++ b/src/serverHandler/aliasHandler.go @@ -5,6 +5,7 @@ import ( "regexp" "strconv" "strings" + "sync/atomic" "mjpclab.dev/ghfs/src/middleware" "mjpclab.dev/ghfs/src/param" @@ -49,7 +50,8 @@ type aliasHandler struct { archive *hierarchyAvailability cors *hierarchyAvailability - archiveWorkers chan struct{} + archiveMaxWorkers int32 + archiveWorkers *int32 globalRestrictAccess []string restrictAccessUrls pathStringsList @@ -143,11 +145,11 @@ func (h *aliasHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func (h *aliasHandler) createArchive(w http.ResponseWriter, r *http.Request, session *sessionContext, data *responseData) bool { - if h.archiveWorkers != nil { - select { - case h.archiveWorkers <- struct{}{}: - defer func() { <-h.archiveWorkers }() - default: + if h.archiveMaxWorkers > 0 { + current := atomic.AddInt32(h.archiveWorkers, -1) + defer atomic.AddInt32(h.archiveWorkers, 1) + + if current < 0 { data.Status = http.StatusTooManyRequests return false } @@ -192,7 +194,8 @@ func newAliasHandler( toHttpsPort: p.ToHttpsPort, defaultSort: p.DefaultSort, - archiveWorkers: p.ArchivationsSem, + archiveMaxWorkers: vhostCtx.archiveMaxWorkers, + archiveWorkers: vhostCtx.archiveWorkers, users: vhostCtx.users, theme: vhostCtx.theme, diff --git a/src/serverHandler/vhostHandler.go b/src/serverHandler/vhostHandler.go index ac3d85f..e177c2d 100644 --- a/src/serverHandler/vhostHandler.go +++ b/src/serverHandler/vhostHandler.go @@ -1,14 +1,15 @@ package serverHandler import ( + "net/http" + "regexp" + "mjpclab.dev/ghfs/src/param" "mjpclab.dev/ghfs/src/serverError" "mjpclab.dev/ghfs/src/serverLog" "mjpclab.dev/ghfs/src/tpl/defaultTheme" "mjpclab.dev/ghfs/src/tpl/theme" "mjpclab.dev/ghfs/src/user" - "net/http" - "regexp" ) type vhostContext struct { @@ -29,6 +30,9 @@ type vhostContext struct { archiveUrlsUsers pathIntsList archiveDirsUsers pathIntsList + archiveMaxWorkers int32 + archiveWorkers *int32 + shows *regexp.Regexp showDirs *regexp.Regexp showFiles *regexp.Regexp @@ -95,6 +99,11 @@ func NewVhostHandler( theme = defaultTheme.DefaultTheme } + var archiveMaxWorkers int32 + if p.ArchiveMaxWorkers > 0 { + archiveMaxWorkers = p.ArchiveMaxWorkers + } + // alias param vhostCtx := &vhostContext{ logger: logger, @@ -114,6 +123,9 @@ func NewVhostHandler( archiveUrlsUsers: pathUsernamesToPathUids(users, p.ArchiveUrlsUsers), archiveDirsUsers: pathUsernamesToPathUids(users, p.ArchiveDirsUsers), + archiveMaxWorkers: p.ArchiveMaxWorkers, + archiveWorkers: &archiveMaxWorkers, + shows: shows, showDirs: showDirs, showFiles: showFiles, From cfc35f8418f9463c2af86f2592088f599137135b Mon Sep 17 00:00:00 2001 From: OlegChuev Date: Fri, 25 Apr 2025 09:53:36 +0300 Subject: [PATCH 4/7] Merge remote-tracking branch 'origin/unstable' into feature/added-archive-worker-limit --- src/serverHandler/archive.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/serverHandler/archive.go b/src/serverHandler/archive.go index df33165..15c8790 100644 --- a/src/serverHandler/archive.go +++ b/src/serverHandler/archive.go @@ -1,12 +1,13 @@ package serverHandler import ( - "mjpclab.dev/ghfs/src/util" "net/http" "net/url" "os" "path" "strings" + + "mjpclab.dev/ghfs/src/util" ) type archiveCallback func(f *os.File, fInfo os.FileInfo, relPath string) error From 9598aba0d671a577f886ea49a62850103ac376ac Mon Sep 17 00:00:00 2001 From: OlegChuev Date: Fri, 25 Apr 2025 19:18:25 +0300 Subject: [PATCH 5/7] refactor(archive): general style polishing --- src/param/cli.go | 3 ++- src/param/main.go | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/param/cli.go b/src/param/cli.go index 8334296..bbb319b 100644 --- a/src/param/cli.go +++ b/src/param/cli.go @@ -470,7 +470,8 @@ func CmdResultsToParams(results []*goNixArgParser.ParseResult) (params Params, e // certificate certFiles, _ := result.GetStrings("certs") keyFiles, _ := result.GetStrings("keys") - param.CertKeyPaths, _ = goVirtualHost.CertsKeysToPairs(certFiles, keyFiles) + param.CertKeyPaths, es = goVirtualHost.CertsKeysToPairs(certFiles, keyFiles) + errs = append(errs, es...) // listen listens, _ := result.GetStrings("listens") diff --git a/src/param/main.go b/src/param/main.go index 3a401c0..e201bab 100644 --- a/src/param/main.go +++ b/src/param/main.go @@ -165,7 +165,6 @@ func (param *Param) Normalize() (errs []error) { param.DeleteDirs = NormalizeFsPaths(param.DeleteDirs) param.ArchiveUrls = NormalizeUrlPaths(param.ArchiveUrls) param.ArchiveDirs = NormalizeFsPaths(param.ArchiveDirs) - param.CorsUrls = NormalizeUrlPaths(param.CorsUrls) param.CorsDirs = NormalizeFsPaths(param.CorsDirs) From 23d393526610df47b912160dcfb2e3bf1eac1c8b Mon Sep 17 00:00:00 2001 From: OlegChuev Date: Fri, 25 Apr 2025 20:28:40 +0300 Subject: [PATCH 6/7] refactor(archive): replaced int32 with *atomic.Uint32 --- README.md | 2 +- src/param/cli.go | 4 ++-- src/param/main.go | 2 +- src/serverHandler/aliasHandler.go | 31 +++++++++++++++++++++++-------- src/serverHandler/vhostHandler.go | 12 ++++-------- 5 files changed, 31 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 1eedf37..14711e3 100644 --- a/README.md +++ b/README.md @@ -260,7 +260,7 @@ ghfs [options] A download link will appear on top part of the page. --archive-max-workers Maximum number of concurrent archive operations. - Set to -1 for unlimited (default). + Set to 0 for unlimited (default). When the limit is reached, new archive requests will receive 429 Too Many Requests. --archive ... --archive-user [...] ... diff --git a/src/param/cli.go b/src/param/cli.go index bbb319b..9cf45c0 100644 --- a/src/param/cli.go +++ b/src/param/cli.go @@ -148,7 +148,7 @@ func NewCliCmd() *goNixArgParser.Command { err = options.AddFlagValues("archivedirsusers", "--archive-dir-user", "", nil, "file system path that allow archive files for specific users, [...]") serverError.CheckFatal(err) - err = options.AddFlagValue("archivemaxworkers", "--archive-max-workers", "", "-1", "maximum number of concurrent archive operations (-1 for unlimited)") + err = options.AddFlagValue("archivemaxworkers", "--archive-max-workers", "", "0", "maximum number of concurrent archive operations (0 for unlimited)") serverError.CheckFatal(err) err = options.AddFlag("globalcors", "--global-cors", "GHFS_GLOBAL_CORS", "enable CORS headers for all directories") @@ -440,7 +440,7 @@ func CmdResultsToParams(results []*goNixArgParser.ParseResult) (params Params, e archiveDirsUsers, _ := result.GetStrings("archivedirsusers") param.ArchiveDirsUsers = SplitAllKeyValues(archiveDirsUsers) - param.ArchiveMaxWorkers, _ = result.GetInt32("archivemaxworkers") + param.ArchiveMaxWorkers, _ = result.GetUint32("archivemaxworkers") // global restrict access if result.HasKey("globalrestrictaccess") { diff --git a/src/param/main.go b/src/param/main.go index e201bab..e3e17b0 100644 --- a/src/param/main.go +++ b/src/param/main.go @@ -62,7 +62,7 @@ type Param struct { ArchiveUrlsUsers [][]string // [][path, user...] ArchiveDirs []string ArchiveDirsUsers [][]string // [][path, user...] - ArchiveMaxWorkers int32 + ArchiveMaxWorkers uint32 GlobalCors bool CorsUrls []string diff --git a/src/serverHandler/aliasHandler.go b/src/serverHandler/aliasHandler.go index 95f4aba..b3333b2 100644 --- a/src/serverHandler/aliasHandler.go +++ b/src/serverHandler/aliasHandler.go @@ -50,8 +50,8 @@ type aliasHandler struct { archive *hierarchyAvailability cors *hierarchyAvailability - archiveMaxWorkers int32 - archiveWorkers *int32 + archiveMaxWorkers uint32 + archiveWorkers *atomic.Uint32 globalRestrictAccess []string restrictAccessUrls pathStringsList @@ -146,13 +146,28 @@ func (h *aliasHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (h *aliasHandler) createArchive(w http.ResponseWriter, r *http.Request, session *sessionContext, data *responseData) bool { if h.archiveMaxWorkers > 0 { - current := atomic.AddInt32(h.archiveWorkers, -1) - defer atomic.AddInt32(h.archiveWorkers, 1) - - if current < 0 { - data.Status = http.StatusTooManyRequests - return false + for { + current := h.archiveWorkers.Load() + if current >= h.archiveMaxWorkers { + data.Status = http.StatusTooManyRequests + return false + } + if h.archiveWorkers.CompareAndSwap(current, current+1) { + break + } } + + defer func() { + for { + current := h.archiveWorkers.Load() + if current == 0 { + break // prevent underflow just in case + } + if h.archiveWorkers.CompareAndSwap(current, current-1) { + break + } + } + }() } switch session.archiveFormat { diff --git a/src/serverHandler/vhostHandler.go b/src/serverHandler/vhostHandler.go index e177c2d..c22226f 100644 --- a/src/serverHandler/vhostHandler.go +++ b/src/serverHandler/vhostHandler.go @@ -3,6 +3,7 @@ package serverHandler import ( "net/http" "regexp" + "sync/atomic" "mjpclab.dev/ghfs/src/param" "mjpclab.dev/ghfs/src/serverError" @@ -30,8 +31,8 @@ type vhostContext struct { archiveUrlsUsers pathIntsList archiveDirsUsers pathIntsList - archiveMaxWorkers int32 - archiveWorkers *int32 + archiveMaxWorkers uint32 + archiveWorkers *atomic.Uint32 shows *regexp.Regexp showDirs *regexp.Regexp @@ -99,11 +100,6 @@ func NewVhostHandler( theme = defaultTheme.DefaultTheme } - var archiveMaxWorkers int32 - if p.ArchiveMaxWorkers > 0 { - archiveMaxWorkers = p.ArchiveMaxWorkers - } - // alias param vhostCtx := &vhostContext{ logger: logger, @@ -124,7 +120,7 @@ func NewVhostHandler( archiveDirsUsers: pathUsernamesToPathUids(users, p.ArchiveDirsUsers), archiveMaxWorkers: p.ArchiveMaxWorkers, - archiveWorkers: &archiveMaxWorkers, + archiveWorkers: &atomic.Uint32{}, shows: shows, showDirs: showDirs, From 63dedd2b8b8749973f254a7eb8126032c7436bea Mon Sep 17 00:00:00 2001 From: marjune Date: Sat, 26 Apr 2025 14:09:25 +0800 Subject: [PATCH 7/7] feat(archive): refine max archive workers mechanism --- README.md | 9 ++--- README.zh-CN.md | 6 +++- src/param/cli.go | 11 +++--- src/param/main.go | 7 ++-- src/serverHandler/aliasHandler.go | 58 +++++-------------------------- src/serverHandler/archive.go | 38 ++++++++++++++++++-- src/serverHandler/vhostHandler.go | 17 ++++----- 7 files changed, 72 insertions(+), 74 deletions(-) diff --git a/README.md b/README.md index 14711e3..8f46006 100644 --- a/README.md +++ b/README.md @@ -258,16 +258,17 @@ ghfs [options] -A|--global-archive Allow user to download the whole contents of current directory for all url paths. A download link will appear on top part of the page. ---archive-max-workers - Maximum number of concurrent archive operations. - Set to 0 for unlimited (default). - When the limit is reached, new archive requests will receive 429 Too Many Requests. + Make sure there is no circular symbol links. --archive ... --archive-user [...] ... Allow user to download the whole contents of current directory for specific url paths(and sub paths). --archive-dir ... --archive-dir-user [...] ... Similar to --archive, but use file system path instead of url path. +--archive-workers-max + Maximum number of concurrent archive operations. + Set to 0 for unlimited (default). + When the limit is reached, new archive requests will be rejected with status code 429(Too Many Requests). --global-cors Allow CORS requests for all url path. diff --git a/README.zh-CN.md b/README.zh-CN.md index a0f5276..2ebc649 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -28,7 +28,7 @@ go build main.go 在8080端口启动服务器，根目录为当前工作目录： ```sh ghfs -l 8080 -``` +``` 在8080端口启动服务器，根目录为 /usr/share/doc： ```sh @@ -255,6 +255,10 @@ ghfs [选项] --archive-dir <文件系统路径> ... --archive-dir-user <分隔符><文件系统路径>[<分隔符><允许的用户名>...] ... 与--archive类似，但指定的是文件系统路径，而不是URL路径。 +--archive-workers-max <数值> + 指定打包下载的最大并发数。 + 设为0（默认）表示无限制。 + 达到并发上限后，会拒绝后续打包请求并返回状态码429（Too Many Requests）。 --global-cors 接受所有URL路径的CORS跨域请求。 diff --git a/src/param/cli.go b/src/param/cli.go index 9cf45c0..bcfd3b6 100644 --- a/src/param/cli.go +++ b/src/param/cli.go @@ -2,13 +2,12 @@ package param import ( "errors" - "net/http" - "os" - "strings" - "mjpclab.dev/ghfs/src/goNixArgParser" "mjpclab.dev/ghfs/src/goVirtualHost" "mjpclab.dev/ghfs/src/serverError" + "net/http" + "os" + "strings" ) var cliCmd = NewCliCmd() @@ -148,7 +147,7 @@ func NewCliCmd() *goNixArgParser.Command { err = options.AddFlagValues("archivedirsusers", "--archive-dir-user", "", nil, "file system path that allow archive files for specific users, [...]") serverError.CheckFatal(err) - err = options.AddFlagValue("archivemaxworkers", "--archive-max-workers", "", "0", "maximum number of concurrent archive operations (0 for unlimited)") + err = options.AddFlagValue("archiveworkersmax", "--archive-workers-max", "", "0", "maximum number of concurrent archive operations (0 for unlimited)") serverError.CheckFatal(err) err = options.AddFlag("globalcors", "--global-cors", "GHFS_GLOBAL_CORS", "enable CORS headers for all directories") @@ -440,7 +439,7 @@ func CmdResultsToParams(results []*goNixArgParser.ParseResult) (params Params, e archiveDirsUsers, _ := result.GetStrings("archivedirsusers") param.ArchiveDirsUsers = SplitAllKeyValues(archiveDirsUsers) - param.ArchiveMaxWorkers, _ = result.GetUint32("archivemaxworkers") + param.ArchiveWorkersMax, _ = result.GetUint32("archiveworkersmax") // global restrict access if result.HasKey("globalrestrictaccess") { diff --git a/src/param/main.go b/src/param/main.go index e3e17b0..0403243 100644 --- a/src/param/main.go +++ b/src/param/main.go @@ -1,12 +1,11 @@ package param import ( - "os" - "path/filepath" - "mjpclab.dev/ghfs/src/middleware" "mjpclab.dev/ghfs/src/serverError" "mjpclab.dev/ghfs/src/util" + "os" + "path/filepath" ) type Param struct { @@ -62,7 +61,7 @@ type Param struct { ArchiveUrlsUsers [][]string // [][path, user...] ArchiveDirs []string ArchiveDirsUsers [][]string // [][path, user...] - ArchiveMaxWorkers uint32 + ArchiveWorkersMax uint32 GlobalCors bool CorsUrls []string diff --git a/src/serverHandler/aliasHandler.go b/src/serverHandler/aliasHandler.go index b3333b2..15d8e93 100644 --- a/src/serverHandler/aliasHandler.go +++ b/src/serverHandler/aliasHandler.go @@ -1,18 +1,16 @@ package serverHandler import ( - "net/http" - "regexp" - "strconv" - "strings" - "sync/atomic" - "mjpclab.dev/ghfs/src/middleware" "mjpclab.dev/ghfs/src/param" "mjpclab.dev/ghfs/src/serverLog" "mjpclab.dev/ghfs/src/tpl/theme" "mjpclab.dev/ghfs/src/user" "mjpclab.dev/ghfs/src/util" + "net/http" + "regexp" + "strconv" + "strings" ) var defaultHandler = http.NotFoundHandler() @@ -50,8 +48,8 @@ type aliasHandler struct { archive *hierarchyAvailability cors *hierarchyAvailability - archiveMaxWorkers uint32 - archiveWorkers *atomic.Uint32 + archiveWorkersMax uint32 + archivingWorkers *uint32 globalRestrictAccess []string restrictAccessUrls pathStringsList @@ -125,7 +123,7 @@ func (h *aliasHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if session.isMutate && h.mutate(w, r, session, data) { return - } else if session.isArchive && h.createArchive(w, r, session, data) { + } else if session.isArchive && h.tryArchive(w, r, session, data) { return } } @@ -144,44 +142,6 @@ func (h *aliasHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } } -func (h *aliasHandler) createArchive(w http.ResponseWriter, r *http.Request, session *sessionContext, data *responseData) bool { - if h.archiveMaxWorkers > 0 { - for { - current := h.archiveWorkers.Load() - if current >= h.archiveMaxWorkers { - data.Status = http.StatusTooManyRequests - return false - } - if h.archiveWorkers.CompareAndSwap(current, current+1) { - break - } - } - - defer func() { - for { - current := h.archiveWorkers.Load() - if current == 0 { - break // prevent underflow just in case - } - if h.archiveWorkers.CompareAndSwap(current, current-1) { - break - } - } - }() - } - - switch session.archiveFormat { - case tarFmt: - return h.tar(w, r, session, data) - case tgzFmt: - return h.tgz(w, r, session, data) - case zipFmt: - return h.zip(w, r, session, data) - } - - return false -} - func newAliasHandler( p *param.Param, vhostCtx *vhostContext, @@ -209,8 +169,8 @@ func newAliasHandler( toHttpsPort: p.ToHttpsPort, defaultSort: p.DefaultSort, - archiveMaxWorkers: vhostCtx.archiveMaxWorkers, - archiveWorkers: vhostCtx.archiveWorkers, + archiveWorkersMax: p.ArchiveWorkersMax, + archivingWorkers: vhostCtx.archivingWorkers, users: vhostCtx.users, theme: vhostCtx.theme, diff --git a/src/serverHandler/archive.go b/src/serverHandler/archive.go index 15c8790..5770bc1 100644 --- a/src/serverHandler/archive.go +++ b/src/serverHandler/archive.go @@ -1,13 +1,13 @@ package serverHandler import ( + "mjpclab.dev/ghfs/src/util" "net/http" "net/url" "os" "path" "strings" - - "mjpclab.dev/ghfs/src/util" + "sync/atomic" ) type archiveCallback func(f *os.File, fInfo os.FileInfo, relPath string) error @@ -211,3 +211,37 @@ func (h *aliasHandler) normalizeArchiveSelections(r *http.Request) ([]string, bo return selections, true } + +func (h *aliasHandler) startArchive(w http.ResponseWriter, r *http.Request, session *sessionContext, data *responseData) (ok bool) { + switch session.archiveFormat { + case tarFmt: + return h.tar(w, r, session, data) + case tgzFmt: + return h.tgz(w, r, session, data) + case zipFmt: + return h.zip(w, r, session, data) + } + + return +} + +func (h *aliasHandler) tryArchive(w http.ResponseWriter, r *http.Request, session *sessionContext, data *responseData) (ok bool) { + if h.archivingWorkers == nil { + return h.startArchive(w, r, session, data) + } + + if *h.archivingWorkers >= h.archiveWorkersMax { + data.Status = http.StatusTooManyRequests + return + } + + archiving := atomic.AddUint32(h.archivingWorkers, 1) + ok = archiving <= h.archiveWorkersMax + if ok { + ok = h.startArchive(w, r, session, data) + } else { + data.Status = http.StatusTooManyRequests + } + atomic.AddUint32(h.archivingWorkers, ^uint32(0)) // archiveWorkers -= 1 + return +} diff --git a/src/serverHandler/vhostHandler.go b/src/serverHandler/vhostHandler.go index c22226f..8e15cad 100644 --- a/src/serverHandler/vhostHandler.go +++ b/src/serverHandler/vhostHandler.go @@ -1,16 +1,14 @@ package serverHandler import ( - "net/http" - "regexp" - "sync/atomic" - "mjpclab.dev/ghfs/src/param" "mjpclab.dev/ghfs/src/serverError" "mjpclab.dev/ghfs/src/serverLog" "mjpclab.dev/ghfs/src/tpl/defaultTheme" "mjpclab.dev/ghfs/src/tpl/theme" "mjpclab.dev/ghfs/src/user" + "net/http" + "regexp" ) type vhostContext struct { @@ -31,8 +29,7 @@ type vhostContext struct { archiveUrlsUsers pathIntsList archiveDirsUsers pathIntsList - archiveMaxWorkers uint32 - archiveWorkers *atomic.Uint32 + archivingWorkers *uint32 shows *regexp.Regexp showDirs *regexp.Regexp @@ -100,6 +97,11 @@ func NewVhostHandler( theme = defaultTheme.DefaultTheme } + var archivingWorkers *uint32 + if p.ArchiveWorkersMax > 0 { + archivingWorkers = new(uint32) + } + // alias param vhostCtx := &vhostContext{ logger: logger, @@ -119,8 +121,7 @@ func NewVhostHandler( archiveUrlsUsers: pathUsernamesToPathUids(users, p.ArchiveUrlsUsers), archiveDirsUsers: pathUsernamesToPathUids(users, p.ArchiveDirsUsers), - archiveMaxWorkers: p.ArchiveMaxWorkers, - archiveWorkers: &atomic.Uint32{}, + archivingWorkers: archivingWorkers, shows: shows, showDirs: showDirs,