From 6f9c278559789066aa831c1df25b0d866103d02d Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Mon, 8 May 2023 19:49:59 +0800 Subject: [PATCH] Rewrite queue (#24505) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # ⚠️ Breaking Many deprecated queue config options are removed (actually, they should have been removed in 1.18/1.19). If you see the fatal message when starting Gitea: "Please update your app.ini to remove deprecated config options", please follow the error messages to remove these options from your app.ini. Example: ``` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].ISSUE_INDEXER_QUEUE_TYPE`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [E] Removed queue option: `[indexer].UPDATE_BUFFER_LEN`. Use new options in `[queue.issue_indexer]` 2023/05/06 19:39:22 [F] Please update your app.ini to remove deprecated config options ``` Many options in `[queue]` are are dropped, including: `WRAP_IF_NECESSARY`, `MAX_ATTEMPTS`, `TIMEOUT`, `WORKERS`, `BLOCK_TIMEOUT`, `BOOST_TIMEOUT`, `BOOST_WORKERS`, they can be removed from app.ini. # The problem The old queue package has some legacy problems: * complexity: I doubt few people could tell how it works. * maintainability: Too many channels and mutex/cond are mixed together, too many different structs/interfaces depends each other. * stability: due to the complexity & maintainability, sometimes there are strange bugs and difficult to debug, and some code doesn't have test (indeed some code is difficult to test because a lot of things are mixed together). * general applicability: although it is called "queue", its behavior is not a well-known queue. * scalability: it doesn't seem easy to make it work with a cluster without breaking its behaviors. It came from some very old code to "avoid breaking", however, its technical debt is too heavy now. It's a good time to introduce a better "queue" package. # The new queue package It keeps using old config and concept as much as possible. * It only contains two major kinds of concepts: * The "base queue": channel, levelqueue, redis * They have the same abstraction, the same interface, and they are tested by the same testing code. * The "WokerPoolQueue", it uses the "base queue" to provide "worker pool" function, calls the "handler" to process the data in the base queue. * The new code doesn't do "PushBack" * Think about a queue with many workers, the "PushBack" can't guarantee the order for re-queued unhandled items, so in new code it just does "normal push" * The new code doesn't do "pause/resume" * The "pause/resume" was designed to handle some handler's failure: eg: document indexer (elasticsearch) is down * If a queue is paused for long time, either the producers blocks or the new items are dropped. * The new code doesn't do such "pause/resume" trick, it's not a common queue's behavior and it doesn't help much. * If there are unhandled items, the "push" function just blocks for a few seconds and then re-queue them and retry. * The new code doesn't do "worker booster" * Gitea's queue's handlers are light functions, the cost is only the go-routine, so it doesn't make sense to "boost" them. * The new code only use "max worker number" to limit the concurrent workers. * The new "Push" never blocks forever * Instead of creating more and more blocking goroutines, return an error is more friendly to the server and to the end user. There are more details in code comments: eg: the "Flush" problem, the strange "code.index" hanging problem, the "immediate" queue problem. Almost ready for review. TODO: * [x] add some necessary comments during review * [x] add some more tests if necessary * [x] update documents and config options * [x] test max worker / active worker * [x] re-run the CI tasks to see whether any test is flaky * [x] improve the `handleOldLengthConfiguration` to provide more friendly messages * [x] fine tune default config values (eg: length?) ## Code coverage: ![image](https://user-images.githubusercontent.com/2114189/236620635-55576955-f95d-4810-b12f-879026a3afdf.png) --- cmd/hook.go | 3 +- custom/conf/app.example.ini | 49 +- .../config-cheat-sheet.en-us.md | 44 +- .../config-cheat-sheet.zh-cn.md | 6 - .../doc/administration/repo-indexer.en-us.md | 1 - models/migrations/base/testlogger.go | 180 ----- models/migrations/base/tests.go | 8 +- models/unittest/testdb.go | 3 + modules/indexer/code/bleve.go | 4 - modules/indexer/code/elastic_search.go | 22 +- modules/indexer/code/indexer.go | 55 +- modules/indexer/code/wrapped.go | 10 - modules/indexer/issues/bleve.go | 4 - modules/indexer/issues/db.go | 4 - modules/indexer/issues/elastic_search.go | 22 +- modules/indexer/issues/indexer.go | 73 +-- modules/indexer/issues/meilisearch.go | 22 +- modules/indexer/stats/indexer_test.go | 2 +- modules/indexer/stats/queue.go | 9 +- modules/mirror/mirror.go | 6 +- modules/notification/ui/ui.go | 11 +- modules/queue/backoff.go | 63 ++ modules/queue/base.go | 42 ++ modules/queue/base_channel.go | 123 ++++ modules/queue/base_channel_test.go | 11 + modules/queue/base_dummy.go | 38 ++ modules/queue/base_levelqueue.go | 72 ++ modules/queue/base_levelqueue_common.go | 92 +++ modules/queue/base_levelqueue_test.go | 23 + modules/queue/base_levelqueue_unique.go | 93 +++ modules/queue/base_redis.go | 135 ++++ modules/queue/base_redis_test.go | 71 ++ modules/queue/base_test.go | 140 ++++ modules/queue/bytefifo.go | 69 -- modules/queue/config.go | 36 + modules/queue/helper.go | 91 --- modules/queue/manager.go | 501 +++----------- modules/queue/manager_test.go | 124 ++++ modules/queue/queue.go | 220 +------ modules/queue/queue_bytefifo.go | 419 ------------ modules/queue/queue_channel.go | 160 ----- modules/queue/queue_channel_test.go | 315 --------- modules/queue/queue_disk.go | 124 ---- modules/queue/queue_disk_channel.go | 358 ---------- modules/queue/queue_disk_channel_test.go | 544 ---------------- modules/queue/queue_disk_test.go | 147 ----- modules/queue/queue_redis.go | 137 ---- modules/queue/queue_test.go | 42 -- modules/queue/queue_wrapped.go | 315 --------- modules/queue/setting.go | 126 ---- modules/queue/testhelper.go | 40 ++ modules/queue/unique_queue.go | 28 - modules/queue/unique_queue_channel.go | 212 ------ modules/queue/unique_queue_channel_test.go | 258 -------- modules/queue/unique_queue_disk.go | 128 ---- modules/queue/unique_queue_disk_channel.go | 336 ---------- .../queue/unique_queue_disk_channel_test.go | 265 -------- modules/queue/unique_queue_redis.go | 141 ---- modules/queue/unique_queue_wrapped.go | 174 ----- modules/queue/workergroup.go | 331 ++++++++++ modules/queue/workerpool.go | 613 ------------------ modules/queue/workerqueue.go | 241 +++++++ modules/queue/workerqueue_test.go | 260 ++++++++ modules/setting/config_provider.go | 6 +- modules/setting/cron_test.go | 2 +- modules/setting/indexer.go | 9 - modules/setting/queue.go | 243 +++---- modules/setting/storage_test.go | 18 +- modules/test/context_tests.go | 1 + {tests => modules/testlogger}/testlogger.go | 81 ++- modules/util/timer.go | 12 - routers/web/admin/admin.go | 178 +---- routers/web/admin/queue.go | 59 ++ routers/web/web.go | 7 +- services/actions/init.go | 2 +- services/actions/job_emitter.go | 11 +- services/automerge/automerge.go | 18 +- services/convert/utils_test.go | 2 - services/mailer/mailer.go | 9 +- services/migrations/github.go | 3 +- services/mirror/mirror.go | 7 +- services/pull/check.go | 35 +- services/pull/check_test.go | 29 +- services/repository/archiver/archiver.go | 15 +- services/repository/push.go | 9 +- services/task/task.go | 9 +- services/webhook/deliver.go | 2 +- services/webhook/webhook.go | 10 +- templates/admin/monitor.tmpl | 31 +- templates/admin/queue.tmpl | 217 +------ templates/admin/queue_manage.tmpl | 48 ++ tests/e2e/e2e_test.go | 3 +- tests/integration/api_branch_test.go | 1 - tests/integration/integration_test.go | 11 +- tests/mssql.ini.tmpl | 2 +- tests/mysql.ini.tmpl | 5 +- tests/mysql8.ini.tmpl | 2 +- tests/pgsql.ini.tmpl | 2 +- tests/sqlite.ini.tmpl | 2 +- tests/test_utils.go | 57 +- 100 files changed, 2496 insertions(+), 6858 deletions(-) delete mode 100644 models/migrations/base/testlogger.go create mode 100644 modules/queue/backoff.go create mode 100644 modules/queue/base.go create mode 100644 modules/queue/base_channel.go create mode 100644 modules/queue/base_channel_test.go create mode 100644 modules/queue/base_dummy.go create mode 100644 modules/queue/base_levelqueue.go create mode 100644 modules/queue/base_levelqueue_common.go create mode 100644 modules/queue/base_levelqueue_test.go create mode 100644 modules/queue/base_levelqueue_unique.go create mode 100644 modules/queue/base_redis.go create mode 100644 modules/queue/base_redis_test.go create mode 100644 modules/queue/base_test.go delete mode 100644 modules/queue/bytefifo.go create mode 100644 modules/queue/config.go delete mode 100644 modules/queue/helper.go create mode 100644 modules/queue/manager_test.go delete mode 100644 modules/queue/queue_bytefifo.go delete mode 100644 modules/queue/queue_channel.go delete mode 100644 modules/queue/queue_channel_test.go delete mode 100644 modules/queue/queue_disk.go delete mode 100644 modules/queue/queue_disk_channel.go delete mode 100644 modules/queue/queue_disk_channel_test.go delete mode 100644 modules/queue/queue_disk_test.go delete mode 100644 modules/queue/queue_redis.go delete mode 100644 modules/queue/queue_test.go delete mode 100644 modules/queue/queue_wrapped.go delete mode 100644 modules/queue/setting.go create mode 100644 modules/queue/testhelper.go delete mode 100644 modules/queue/unique_queue.go delete mode 100644 modules/queue/unique_queue_channel.go delete mode 100644 modules/queue/unique_queue_channel_test.go delete mode 100644 modules/queue/unique_queue_disk.go delete mode 100644 modules/queue/unique_queue_disk_channel.go delete mode 100644 modules/queue/unique_queue_disk_channel_test.go delete mode 100644 modules/queue/unique_queue_redis.go delete mode 100644 modules/queue/unique_queue_wrapped.go create mode 100644 modules/queue/workergroup.go delete mode 100644 modules/queue/workerpool.go create mode 100644 modules/queue/workerqueue.go create mode 100644 modules/queue/workerqueue_test.go rename {tests => modules/testlogger}/testlogger.go (77%) create mode 100644 routers/web/admin/queue.go create mode 100644 templates/admin/queue_manage.tmpl diff --git a/cmd/hook.go b/cmd/hook.go index 9605fcb33..bd5575ab6 100644 --- a/cmd/hook.go +++ b/cmd/hook.go @@ -18,7 +18,6 @@ import ( "code.gitea.io/gitea/modules/private" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" - "code.gitea.io/gitea/modules/util" "github.com/urfave/cli" ) @@ -141,7 +140,7 @@ func (d *delayWriter) Close() error { if d == nil { return nil } - stopped := util.StopTimer(d.timer) + stopped := d.timer.Stop() if stopped || d.buf == nil { return nil } diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 108d94089..95ad9fe39 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -926,12 +926,6 @@ ROUTER = console ;; Global limit of repositories per user, applied at creation time. -1 means no limit ;MAX_CREATION_LIMIT = -1 ;; -;; Mirror sync queue length, increase if mirror syncing starts hanging (DEPRECATED: please use [queue.mirror] LENGTH instead) -;MIRROR_QUEUE_LENGTH = 1000 -;; -;; Patch test queue length, increase if pull request patch testing starts hanging (DEPRECATED: please use [queue.pr_patch_checker] LENGTH instead) -;PULL_REQUEST_QUEUE_LENGTH = 1000 -;; ;; Preferred Licenses to place at the top of the List ;; The name here must match the filename in options/license or custom/options/license ;PREFERRED_LICENSES = Apache License 2.0,MIT License @@ -1376,22 +1370,6 @@ ROUTER = console ;; Set to -1 to disable timeout. ;STARTUP_TIMEOUT = 30s ;; -;; Issue indexer queue, currently support: channel, levelqueue or redis, default is levelqueue (deprecated - use [queue.issue_indexer]) -;ISSUE_INDEXER_QUEUE_TYPE = levelqueue; **DEPRECATED** use settings in `[queue.issue_indexer]`. -;; -;; When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the path where the queue will be saved. -;; This can be overridden by `ISSUE_INDEXER_QUEUE_CONN_STR`. -;; default is queues/common -;ISSUE_INDEXER_QUEUE_DIR = queues/common; **DEPRECATED** use settings in `[queue.issue_indexer]`. Relative paths will be made absolute against `%(APP_DATA_PATH)s`. -;; -;; When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. -;; When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of -;; the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`. -;ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"; **DEPRECATED** use settings in `[queue.issue_indexer]`. -;; -;; Batch queue number, default is 20 -;ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`. - ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Repository Indexer settings ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1418,8 +1396,6 @@ ROUTER = console ;; A comma separated list of glob patterns to exclude from the index; ; default is empty ;REPO_INDEXER_EXCLUDE = ;; -;; -;UPDATE_BUFFER_LEN = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`. ;MAX_FILE_SIZE = 1048576 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; @@ -1441,7 +1417,7 @@ ROUTER = console ;DATADIR = queues/ ; Relative paths will be made absolute against `%(APP_DATA_PATH)s`. ;; ;; Default queue length before a channel queue will block -;LENGTH = 20 +;LENGTH = 100 ;; ;; Batch size to send for batched queues ;BATCH_LENGTH = 20 @@ -1449,7 +1425,7 @@ ROUTER = console ;; Connection string for redis queues this will store the redis connection string. ;; When `TYPE` is `persistable-channel`, this provides a directory for the underlying leveldb ;; or additional options of the form `leveldb://path/to/db?option=value&....`, and will override `DATADIR`. -;CONN_STR = "addrs=127.0.0.1:6379 db=0" +;CONN_STR = "redis://127.0.0.1:6379/0" ;; ;; Provides the suffix of the default redis/disk queue name - specific queues can be overridden within in their [queue.name] sections. ;QUEUE_NAME = "_queue" @@ -1457,29 +1433,8 @@ ROUTER = console ;; Provides the suffix of the default redis/disk unique queue set name - specific queues can be overridden within in their [queue.name] sections. ;SET_NAME = "_unique" ;; -;; If the queue cannot be created at startup - level queues may need a timeout at startup - wrap the queue: -;WRAP_IF_NECESSARY = true -;; -;; Attempt to create the wrapped queue at max -;MAX_ATTEMPTS = 10 -;; -;; Timeout queue creation -;TIMEOUT = 15m30s -;; -;; Create a pool with this many workers -;WORKERS = 0 -;; ;; Dynamically scale the worker pool to at this many workers ;MAX_WORKERS = 10 -;; -;; Add boost workers when the queue blocks for BLOCK_TIMEOUT -;BLOCK_TIMEOUT = 1s -;; -;; Remove the boost workers after BOOST_TIMEOUT -;BOOST_TIMEOUT = 5m -;; -;; During a boost add BOOST_WORKERS -;BOOST_WORKERS = 1 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index 727d011cf..294f50bc8 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -89,10 +89,6 @@ In addition there is _`StaticRootPath`_ which can be set as a built-in at build - `DEFAULT_PUSH_CREATE_PRIVATE`: **true**: Default private when creating a new repository with push-to-create. - `MAX_CREATION_LIMIT`: **-1**: Global maximum creation limit of repositories per user, `-1` means no limit. -- `PULL_REQUEST_QUEUE_LENGTH`: **1000**: Length of pull request patch test queue, make it. **DEPRECATED** use `LENGTH` in `[queue.pr_patch_checker]`. - as large as possible. Use caution when editing this value. -- `MIRROR_QUEUE_LENGTH`: **1000**: Patch test queue length, increase if pull request patch - testing starts hanging. **DEPRECATED** use `LENGTH` in `[queue.mirror]`. - `PREFERRED_LICENSES`: **Apache License 2.0,MIT License**: Preferred Licenses to place at the top of the list. Name must match file name in options/license or custom/options/license. - `DISABLE_HTTP_GIT`: **false**: Disable the ability to interact with repositories over the @@ -465,11 +461,6 @@ relation to port exhaustion. - `ISSUE_INDEXER_CONN_STR`: ****: Issue indexer connection string, available when ISSUE_INDEXER_TYPE is elasticsearch, or meilisearch. i.e. http://elastic:changeme@localhost:9200 - `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch. Relative paths will be made absolute against _`AppWorkPath`_. -- The next 4 configuration values are deprecated and should be set in `queue.issue_indexer` however are kept for backwards compatibility: -- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`. **DEPRECATED** use settings in `[queue.issue_indexer]`. -- `ISSUE_INDEXER_QUEUE_DIR`: **queues/common**: When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this will be the path where the queue will be saved. **DEPRECATED** use settings in `[queue.issue_indexer]`. Relative paths will be made absolute against `%(APP_DATA_PATH)s`. -- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`. **DEPRECATED** use settings in `[queue.issue_indexer]`. -- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number. **DEPRECATED** use settings in `[queue.issue_indexer]`. - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size). - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`. @@ -480,7 +471,6 @@ relation to port exhaustion. - `REPO_INDEXER_INCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **include** in the index. Use `**.txt` to match any files with .txt extension. An empty list means include all files. - `REPO_INDEXER_EXCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **exclude** from the index. Files that match this list will not be indexed, even if they match in `REPO_INDEXER_INCLUDE`. - `REPO_INDEXER_EXCLUDE_VENDORED`: **true**: Exclude vendored files from index. -- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. **DEPRECATED** use settings in `[queue.issue_indexer]`. - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed. - `STARTUP_TIMEOUT`: **30s**: If the indexer takes longer than this timeout to start - fail. (This timeout will be added to the hammer time above for child processes - as bleve will not start until the previous parent is shutdown.) Set to -1 to never timeout. @@ -488,23 +478,14 @@ relation to port exhaustion. Configuration at `[queue]` will set defaults for queues with overrides for individual queues at `[queue.*]`. (However see below.) -- `TYPE`: **persistable-channel**: General queue type, currently support: `persistable-channel` (uses a LevelDB internally), `channel`, `level`, `redis`, `dummy` -- `DATADIR`: **queues/**: Base DataDir for storing persistent and level queues. `DATADIR` for individual queues can be set in `queue.name` sections but will default to `DATADIR/`**`common`**. (Previously each queue would default to `DATADIR/`**`name`**.) Relative paths will be made absolute against `%(APP_DATA_PATH)s`. -- `LENGTH`: **20**: Maximal queue size before channel queues block +- `TYPE`: **level**: General queue type, currently support: `level` (uses a LevelDB internally), `channel`, `redis`, `dummy`. Invalid types are treated as `level`. +- `DATADIR`: **queues/common**: Base DataDir for storing level queues. `DATADIR` for individual queues can be set in `queue.name` sections. Relative paths will be made absolute against `%(APP_DATA_PATH)s`. +- `LENGTH`: **100**: Maximal queue size before channel queues block - `BATCH_LENGTH`: **20**: Batch data before passing to the handler -- `CONN_STR`: **redis://127.0.0.1:6379/0**: Connection string for the redis queue type. Options can be set using query params. Similarly LevelDB options can also be set using: **leveldb://relative/path?option=value** or **leveldb:///absolute/path?option=value**, and will override `DATADIR` +- `CONN_STR`: **redis://127.0.0.1:6379/0**: Connection string for the redis queue type. Options can be set using query params. Similarly, LevelDB options can also be set using: **leveldb://relative/path?option=value** or **leveldb:///absolute/path?option=value**, and will override `DATADIR` - `QUEUE_NAME`: **_queue**: The suffix for default redis and disk queue name. Individual queues will default to **`name`**`QUEUE_NAME` but can be overridden in the specific `queue.name` section. -- `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to - **`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section. -- `WRAP_IF_NECESSARY`: **true**: Will wrap queues with a timeoutable queue if the selected queue is not ready to be created - (Only relevant for the level queue.) -- `MAX_ATTEMPTS`: **10**: Maximum number of attempts to create the wrapped queue -- `TIMEOUT`: **GRACEFUL_HAMMER_TIME + 30s**: Timeout the creation of the wrapped queue if it takes longer than this to create. -- Queues by default come with a dynamically scaling worker pool. The following settings configure this: -- `WORKERS`: **0**: Number of initial workers for the queue. +- `SET_NAME`: **_unique**: The suffix that will be added to the default redis and disk queue `set` name for unique queues. Individual queues will default to **`name`**`QUEUE_NAME`_`SET_NAME`_ but can be overridden in the specific `queue.name` section. - `MAX_WORKERS`: **10**: Maximum number of worker go-routines for the queue. -- `BLOCK_TIMEOUT`: **1s**: If the queue blocks for this time, boost the number of workers - the `BLOCK_TIMEOUT` will then be doubled before boosting again whilst the boost is ongoing. -- `BOOST_TIMEOUT`: **5m**: Boost workers will timeout after this long. -- `BOOST_WORKERS`: **1**: This many workers will be added to the worker pool if there is a boost. Gitea creates the following non-unique queues: @@ -522,21 +503,6 @@ And the following unique queues: - `mirror` - `pr_patch_checker` -Certain queues have defaults that override the defaults set in `[queue]` (this occurs mostly to support older configuration): - -- `[queue.issue_indexer]` - - `TYPE` this will default to `[queue]` `TYPE` if it is set but if not it will appropriately convert `[indexer]` `ISSUE_INDEXER_QUEUE_TYPE` if that is set. - - `LENGTH` will default to `[indexer]` `UPDATE_BUFFER_LEN` if that is set. - - `BATCH_LENGTH` will default to `[indexer]` `ISSUE_INDEXER_QUEUE_BATCH_NUMBER` if that is set. - - `DATADIR` will default to `[indexer]` `ISSUE_INDEXER_QUEUE_DIR` if that is set. - - `CONN_STR` will default to `[indexer]` `ISSUE_INDEXER_QUEUE_CONN_STR` if that is set. -- `[queue.mailer]` - - `LENGTH` will default to **100** or whatever `[mailer]` `SEND_BUFFER_LEN` is. -- `[queue.pr_patch_checker]` - - `LENGTH` will default to **1000** or whatever `[repository]` `PULL_REQUEST_QUEUE_LENGTH` is. -- `[queue.mirror]` - - `LENGTH` will default to **1000** or whatever `[repository]` `MIRROR_QUEUE_LENGTH` is. - ## Admin (`admin`) - `DEFAULT_EMAIL_NOTIFICATIONS`: **enabled**: Default configuration for email notifications for users (user configurable). Options: enabled, onmention, disabled diff --git a/docs/content/doc/administration/config-cheat-sheet.zh-cn.md b/docs/content/doc/administration/config-cheat-sheet.zh-cn.md index eb015908b..41eed612a 100644 --- a/docs/content/doc/administration/config-cheat-sheet.zh-cn.md +++ b/docs/content/doc/administration/config-cheat-sheet.zh-cn.md @@ -43,7 +43,6 @@ menu: - `DEFAULT_PRIVATE`: 默认创建的git工程为私有。 可以是`last`, `private` 或 `public`。默认值是 `last`表示用户最后创建的Repo的选择。 - `DEFAULT_PUSH_CREATE_PRIVATE`: **true**: 通过 ``push-to-create`` 方式创建的仓库是否默认为私有仓库. - `MAX_CREATION_LIMIT`: 全局最大每个用户创建的git工程数目， `-1` 表示没限制。 -- `PULL_REQUEST_QUEUE_LENGTH`: 小心：合并请求测试队列的长度，尽量放大。 ### Repository - Release (`repository.release`) @@ -111,10 +110,6 @@ menu: - `ISSUE_INDEXER_CONN_STR`: ****: 工单索引连接字符串，仅当 ISSUE_INDEXER_TYPE 为 `elasticsearch` 时有效。例如: http://elastic:changeme@localhost:9200 - `ISSUE_INDEXER_NAME`: **gitea_issues**: 工单索引名称，仅当 ISSUE_INDEXER_TYPE 为 `elasticsearch` 时有效。 - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: 工单索引文件存放路径，当索引类型为 `bleve` 时有效。 -- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: 工单索引队列类型，当前支持 `channel`， `levelqueue` 或 `redis`。 -- `ISSUE_INDEXER_QUEUE_DIR`: **indexers/issues.queue**: 当 `ISSUE_INDEXER_QUEUE_TYPE` 为 `levelqueue` 时，保存索引队列的磁盘路径。 -- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: 当 `ISSUE_INDEXER_QUEUE_TYPE` 为 `redis` 时，保存Redis队列的连接字符串。 -- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: 队列处理中批量提交数量。 - `REPO_INDEXER_ENABLED`: **false**: 是否启用代码搜索（启用后会占用比较大的磁盘空间，如果是bleve可能需要占用约6倍存储空间）。 - `REPO_INDEXER_TYPE`: **bleve**: 代码搜索引擎类型，可以为 `bleve` 或者 `elasticsearch`。 @@ -122,7 +117,6 @@ menu: - `REPO_INDEXER_CONN_STR`: ****: 代码搜索引擎连接字符串，当 `REPO_INDEXER_TYPE` 为 `elasticsearch` 时有效。例如： http://elastic:changeme@localhost:9200 - `REPO_INDEXER_NAME`: **gitea_codes**: 代码搜索引擎的名字，当 `REPO_INDEXER_TYPE` 为 `elasticsearch` 时有效。 -- `UPDATE_BUFFER_LEN`: **20**: 代码索引请求的缓冲区长度。 - `MAX_FILE_SIZE`: **1048576**: 进行解析的源代码文件的最大长度，小于该值时才会索引。 ## Security (`security`) diff --git a/docs/content/doc/administration/repo-indexer.en-us.md b/docs/content/doc/administration/repo-indexer.en-us.md index 81d224347..a1980bc5f 100644 --- a/docs/content/doc/administration/repo-indexer.en-us.md +++ b/docs/content/doc/administration/repo-indexer.en-us.md @@ -30,7 +30,6 @@ Gitea can search through the files of the repositories by enabling this function ; ... REPO_INDEXER_ENABLED = true REPO_INDEXER_PATH = indexers/repos.bleve -UPDATE_BUFFER_LEN = 20 MAX_FILE_SIZE = 1048576 REPO_INDEXER_INCLUDE = REPO_INDEXER_EXCLUDE = resources/bin/** diff --git a/models/migrations/base/testlogger.go b/models/migrations/base/testlogger.go deleted file mode 100644 index 80e672952..000000000 --- a/models/migrations/base/testlogger.go +++ /dev/null @@ -1,180 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package base - -import ( - "context" - "fmt" - "os" - "runtime" - "strings" - "sync" - "testing" - "time" - - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/queue" -) - -var ( - prefix string - slowTest = 10 * time.Second - slowFlush = 5 * time.Second -) - -// TestLogger is a logger which will write to the testing log -type TestLogger struct { - log.WriterLogger -} - -var writerCloser = &testLoggerWriterCloser{} - -type testLoggerWriterCloser struct { - sync.RWMutex - t []*testing.TB -} - -func (w *testLoggerWriterCloser) setT(t *testing.TB) { - w.Lock() - w.t = append(w.t, t) - w.Unlock() -} - -func (w *testLoggerWriterCloser) Write(p []byte) (int, error) { - w.RLock() - var t *testing.TB - if len(w.t) > 0 { - t = w.t[len(w.t)-1] - } - w.RUnlock() - if t != nil && *t != nil { - if len(p) > 0 && p[len(p)-1] == '\n' { - p = p[:len(p)-1] - } - - defer func() { - err := recover() - if err == nil { - return - } - var errString string - errErr, ok := err.(error) - if ok { - errString = errErr.Error() - } else { - errString, ok = err.(string) - } - if !ok { - panic(err) - } - if !strings.HasPrefix(errString, "Log in goroutine after ") { - panic(err) - } - }() - - (*t).Log(string(p)) - return len(p), nil - } - return len(p), nil -} - -func (w *testLoggerWriterCloser) Close() error { - w.Lock() - if len(w.t) > 0 { - w.t = w.t[:len(w.t)-1] - } - w.Unlock() - return nil -} - -// PrintCurrentTest prints the current test to os.Stdout -func PrintCurrentTest(t testing.TB, skip ...int) func() { - start := time.Now() - actualSkip := 1 - if len(skip) > 0 { - actualSkip = skip[0] - } - _, filename, line, _ := runtime.Caller(actualSkip) - - if log.CanColorStdout { - fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", fmt.Formatter(log.NewColoredValue(t.Name())), strings.TrimPrefix(filename, prefix), line) - } else { - fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", t.Name(), strings.TrimPrefix(filename, prefix), line) - } - writerCloser.setT(&t) - return func() { - took := time.Since(start) - if took > slowTest { - if log.CanColorStdout { - fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgYellow)), fmt.Formatter(log.NewColoredValue(took, log.Bold, log.FgYellow))) - } else { - fmt.Fprintf(os.Stdout, "+++ %s is a slow test (took %v)\n", t.Name(), took) - } - } - timer := time.AfterFunc(slowFlush, func() { - if log.CanColorStdout { - fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), slowFlush) - } else { - fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), slowFlush) - } - }) - if err := queue.GetManager().FlushAll(context.Background(), -1); err != nil { - t.Errorf("Flushing queues failed with error %v", err) - } - timer.Stop() - flushTook := time.Since(start) - took - if flushTook > slowFlush { - if log.CanColorStdout { - fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", fmt.Formatter(log.NewColoredValue(t.Name(), log.Bold, log.FgRed)), fmt.Formatter(log.NewColoredValue(flushTook, log.Bold, log.FgRed))) - } else { - fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", t.Name(), flushTook) - } - } - _ = writerCloser.Close() - } -} - -// Printf takes a format and args and prints the string to os.Stdout -func Printf(format string, args ...interface{}) { - if log.CanColorStdout { - for i := 0; i < len(args); i++ { - args[i] = log.NewColoredValue(args[i]) - } - } - fmt.Fprintf(os.Stdout, "\t"+format, args...) -} - -// NewTestLogger creates a TestLogger as a log.LoggerProvider -func NewTestLogger() log.LoggerProvider { - logger := &TestLogger{} - logger.Colorize = log.CanColorStdout - logger.Level = log.TRACE - return logger -} - -// Init inits connection writer with json config. -// json config only need key "level". -func (log *TestLogger) Init(config string) error { - err := json.Unmarshal([]byte(config), log) - if err != nil { - return err - } - log.NewWriterLogger(writerCloser) - return nil -} - -// Flush when log should be flushed -func (log *TestLogger) Flush() { -} - -// ReleaseReopen does nothing -func (log *TestLogger) ReleaseReopen() error { - return nil -} - -// GetName returns the default name for this implementation -func (log *TestLogger) GetName() string { - return "test" -} diff --git a/models/migrations/base/tests.go b/models/migrations/base/tests.go index 124111f51..08eded7fd 100644 --- a/models/migrations/base/tests.go +++ b/models/migrations/base/tests.go @@ -11,7 +11,6 @@ import ( "path" "path/filepath" "runtime" - "strings" "testing" "code.gitea.io/gitea/models/unittest" @@ -19,6 +18,7 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/testlogger" "github.com/stretchr/testify/assert" "xorm.io/xorm" @@ -32,7 +32,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En t.Helper() ourSkip := 2 ourSkip += skip - deferFn := PrintCurrentTest(t, ourSkip) + deferFn := testlogger.PrintCurrentTest(t, ourSkip) assert.NoError(t, os.RemoveAll(setting.RepoRootPath)) assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) @@ -110,9 +110,7 @@ func PrepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En } func MainTest(m *testing.M) { - log.Register("test", NewTestLogger) - _, filename, _, _ := runtime.Caller(0) - prefix = strings.TrimSuffix(filename, "tests/testlogger.go") + log.Register("test", testlogger.NewTestLogger) giteaRoot := base.SetupGiteaRoot() if giteaRoot == "" { diff --git a/models/unittest/testdb.go b/models/unittest/testdb.go index a5b126350..10a70ad9f 100644 --- a/models/unittest/testdb.go +++ b/models/unittest/testdb.go @@ -202,6 +202,9 @@ type FixturesOptions struct { func CreateTestEngine(opts FixturesOptions) error { x, err := xorm.NewEngine("sqlite3", "file::memory:?cache=shared&_txlock=immediate") if err != nil { + if strings.Contains(err.Error(), "unknown driver") { + return fmt.Errorf(`sqlite3 requires: import _ "github.com/mattn/go-sqlite3" or -tags sqlite,sqlite_unlock_notify%s%w`, "\n", err) + } return err } x.SetMapper(names.GonicMapper{}) diff --git a/modules/indexer/code/bleve.go b/modules/indexer/code/bleve.go index e9085f410..5936613e3 100644 --- a/modules/indexer/code/bleve.go +++ b/modules/indexer/code/bleve.go @@ -273,10 +273,6 @@ func (b *BleveIndexer) Close() { log.Info("PID: %d Repository Indexer closed", os.Getpid()) } -// SetAvailabilityChangeCallback does nothing -func (b *BleveIndexer) SetAvailabilityChangeCallback(callback func(bool)) { -} - // Ping does nothing func (b *BleveIndexer) Ping() bool { return true diff --git a/modules/indexer/code/elastic_search.go b/modules/indexer/code/elastic_search.go index 68c809675..609753800 100644 --- a/modules/indexer/code/elastic_search.go +++ b/modules/indexer/code/elastic_search.go @@ -42,12 +42,11 @@ var _ Indexer = &ElasticSearchIndexer{} // ElasticSearchIndexer implements Indexer interface type ElasticSearchIndexer struct { - client *elastic.Client - indexerAliasName string - available bool - availabilityCallback func(bool) - stopTimer chan struct{} - lock sync.RWMutex + client *elastic.Client + indexerAliasName string + available bool + stopTimer chan struct{} + lock sync.RWMutex } type elasticLogger struct { @@ -198,13 +197,6 @@ func (b *ElasticSearchIndexer) init() (bool, error) { return exists, nil } -// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes -func (b *ElasticSearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) { - b.lock.Lock() - defer b.lock.Unlock() - b.availabilityCallback = callback -} - // Ping checks if elastic is available func (b *ElasticSearchIndexer) Ping() bool { b.lock.RLock() @@ -529,8 +521,4 @@ func (b *ElasticSearchIndexer) setAvailability(available bool) { } b.available = available - if b.availabilityCallback != nil { - // Call the callback from within the lock to ensure that the ordering remains correct - b.availabilityCallback(b.available) - } } diff --git a/modules/indexer/code/indexer.go b/modules/indexer/code/indexer.go index 2c493ccf9..a5e40b52c 100644 --- a/modules/indexer/code/indexer.go +++ b/modules/indexer/code/indexer.go @@ -44,7 +44,6 @@ type SearchResultLanguages struct { // Indexer defines an interface to index and search code contents type Indexer interface { Ping() bool - SetAvailabilityChangeCallback(callback func(bool)) Index(ctx context.Context, repo *repo_model.Repository, sha string, changes *repoChanges) error Delete(repoID int64) error Search(ctx context.Context, repoIDs []int64, language, keyword string, page, pageSize int, isMatch bool) (int64, []*SearchResult, []*SearchResultLanguages, error) @@ -81,7 +80,7 @@ type IndexerData struct { RepoID int64 } -var indexerQueue queue.UniqueQueue +var indexerQueue *queue.WorkerPoolQueue[*IndexerData] func index(ctx context.Context, indexer Indexer, repoID int64) error { repo, err := repo_model.GetRepositoryByID(ctx, repoID) @@ -137,37 +136,45 @@ func Init() { // Create the Queue switch setting.Indexer.RepoType { case "bleve", "elasticsearch": - handler := func(data ...queue.Data) []queue.Data { + handler := func(items ...*IndexerData) (unhandled []*IndexerData) { idx, err := indexer.get() if idx == nil || err != nil { log.Error("Codes indexer handler: unable to get indexer!") - return data + return items } - unhandled := make([]queue.Data, 0, len(data)) - for _, datum := range data { - indexerData, ok := datum.(*IndexerData) - if !ok { - log.Error("Unable to process provided datum: %v - not possible to cast to IndexerData", datum) - continue - } + for _, indexerData := range items { log.Trace("IndexerData Process Repo: %d", indexerData.RepoID) + // FIXME: it seems there is a bug in `CatFileBatch` or `nio.Pipe`, which will cause the process to hang forever in rare cases + /* + sync.(*Cond).Wait(cond.go:70) + github.com/djherbis/nio/v3.(*PipeReader).Read(sync.go:106) + bufio.(*Reader).fill(bufio.go:106) + bufio.(*Reader).ReadSlice(bufio.go:372) + bufio.(*Reader).collectFragments(bufio.go:447) + bufio.(*Reader).ReadString(bufio.go:494) + code.gitea.io/gitea/modules/git.ReadBatchLine(batch_reader.go:149) + code.gitea.io/gitea/modules/indexer/code.(*BleveIndexer).addUpdate(bleve.go:214) + code.gitea.io/gitea/modules/indexer/code.(*BleveIndexer).Index(bleve.go:296) + code.gitea.io/gitea/modules/indexer/code.(*wrappedIndexer).Index(wrapped.go:74) + code.gitea.io/gitea/modules/indexer/code.index(indexer.go:105) + */ if err := index(ctx, indexer, indexerData.RepoID); err != nil { - if !setting.IsInTesting { - log.Error("indexer index error for repo %v: %v", indexerData.RepoID, err) - } - if indexer.Ping() { + if !idx.Ping() { + log.Error("Code indexer handler: indexer is unavailable.") + unhandled = append(unhandled, indexerData) continue } - // Add back to queue - unhandled = append(unhandled, datum) + if !setting.IsInTesting { + log.Error("Codes indexer handler: index error for repo %v: %v", indexerData.RepoID, err) + } } } return unhandled } - indexerQueue = queue.CreateUniqueQueue("code_indexer", handler, &IndexerData{}) + indexerQueue = queue.CreateUniqueQueue("code_indexer", handler) if indexerQueue == nil { log.Fatal("Unable to create codes indexer queue") } @@ -224,18 +231,6 @@ func Init() { indexer.set(rIndexer) - if queue, ok := indexerQueue.(queue.Pausable); ok { - rIndexer.SetAvailabilityChangeCallback(func(available bool) { - if !available { - log.Info("Code index queue paused") - queue.Pause() - } else { - log.Info("Code index queue resumed") - queue.Resume() - } - }) - } - // Start processing the queue go graceful.GetManager().RunWithShutdownFns(indexerQueue.Run) diff --git a/modules/indexer/code/wrapped.go b/modules/indexer/code/wrapped.go index 33ba57a09..7eed3e855 100644 --- a/modules/indexer/code/wrapped.go +++ b/modules/indexer/code/wrapped.go @@ -56,16 +56,6 @@ func (w *wrappedIndexer) get() (Indexer, error) { return w.internal, nil } -// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes -func (w *wrappedIndexer) SetAvailabilityChangeCallback(callback func(bool)) { - indexer, err := w.get() - if err != nil { - log.Error("Failed to get indexer: %v", err) - return - } - indexer.SetAvailabilityChangeCallback(callback) -} - // Ping checks if elastic is available func (w *wrappedIndexer) Ping() bool { indexer, err := w.get() diff --git a/modules/indexer/issues/bleve.go b/modules/indexer/issues/bleve.go index e3ef9af5b..60d9ef761 100644 --- a/modules/indexer/issues/bleve.go +++ b/modules/indexer/issues/bleve.go @@ -187,10 +187,6 @@ func (b *BleveIndexer) Init() (bool, error) { return false, err } -// SetAvailabilityChangeCallback does nothing -func (b *BleveIndexer) SetAvailabilityChangeCallback(callback func(bool)) { -} - // Ping does nothing func (b *BleveIndexer) Ping() bool { return true diff --git a/modules/indexer/issues/db.go b/modules/indexer/issues/db.go index d28b536e0..04c101c35 100644 --- a/modules/indexer/issues/db.go +++ b/modules/indexer/issues/db.go @@ -18,10 +18,6 @@ func (i *DBIndexer) Init() (bool, error) { return false, nil } -// SetAvailabilityChangeCallback dummy function -func (i *DBIndexer) SetAvailabilityChangeCallback(callback func(bool)) { -} - // Ping checks if database is available func (i *DBIndexer) Ping() bool { return db.GetEngine(db.DefaultContext).Ping() != nil diff --git a/modules/indexer/issues/elastic_search.go b/modules/indexer/issues/elastic_search.go index ee8e3df62..fd1dd4b45 100644 --- a/modules/indexer/issues/elastic_search.go +++ b/modules/indexer/issues/elastic_search.go @@ -22,12 +22,11 @@ var _ Indexer = &ElasticSearchIndexer{} // ElasticSearchIndexer implements Indexer interface type ElasticSearchIndexer struct { - client *elastic.Client - indexerName string - available bool - availabilityCallback func(bool) - stopTimer chan struct{} - lock sync.RWMutex + client *elastic.Client + indexerName string + available bool + stopTimer chan struct{} + lock sync.RWMutex } type elasticLogger struct { @@ -138,13 +137,6 @@ func (b *ElasticSearchIndexer) Init() (bool, error) { return true, nil } -// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes -func (b *ElasticSearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) { - b.lock.Lock() - defer b.lock.Unlock() - b.availabilityCallback = callback -} - // Ping checks if elastic is available func (b *ElasticSearchIndexer) Ping() bool { b.lock.RLock() @@ -305,8 +297,4 @@ func (b *ElasticSearchIndexer) setAvailability(available bool) { } b.available = available - if b.availabilityCallback != nil { - // Call the callback from within the lock to ensure that the ordering remains correct - b.availabilityCallback(b.available) - } } diff --git a/modules/indexer/issues/indexer.go b/modules/indexer/issues/indexer.go index 47a8b1079..e88b1b2be 100644 --- a/modules/indexer/issues/indexer.go +++ b/modules/indexer/issues/indexer.go @@ -49,7 +49,6 @@ type SearchResult struct { type Indexer interface { Init() (bool, error) Ping() bool - SetAvailabilityChangeCallback(callback func(bool)) Index(issue []*IndexerData) error Delete(ids ...int64) error Search(ctx context.Context, kw string, repoIDs []int64, limit, start int) (*SearchResult, error) @@ -94,7 +93,7 @@ func (h *indexerHolder) get() Indexer { var ( // issueIndexerQueue queue of issue ids to be updated - issueIndexerQueue queue.Queue + issueIndexerQueue *queue.WorkerPoolQueue[*IndexerData] holder = newIndexerHolder() ) @@ -108,62 +107,44 @@ func InitIssueIndexer(syncReindex bool) { // Create the Queue switch setting.Indexer.IssueType { case "bleve", "elasticsearch", "meilisearch": - handler := func(data ...queue.Data) []queue.Data { + handler := func(items ...*IndexerData) (unhandled []*IndexerData) { indexer := holder.get() if indexer == nil { - log.Error("Issue indexer handler: unable to get indexer!") - return data + log.Error("Issue indexer handler: unable to get indexer.") + return items } - - iData := make([]*IndexerData, 0, len(data)) - unhandled := make([]queue.Data, 0, len(data)) - for _, datum := range data { - indexerData, ok := datum.(*IndexerData) - if !ok { - log.Error("Unable to process provided datum: %v - not possible to cast to IndexerData", datum) - continue - } + toIndex := make([]*IndexerData, 0, len(items)) + for _, indexerData := range items { log.Trace("IndexerData Process: %d %v %t", indexerData.ID, indexerData.IDs, indexerData.IsDelete) if indexerData.IsDelete { if err := indexer.Delete(indexerData.IDs...); err != nil { - log.Error("Error whilst deleting from index: %v Error: %v", indexerData.IDs, err) - if indexer.Ping() { - continue + log.Error("Issue indexer handler: failed to from index: %v Error: %v", indexerData.IDs, err) + if !indexer.Ping() { + log.Error("Issue indexer handler: indexer is unavailable when deleting") + unhandled = append(unhandled, indexerData) } - // Add back to queue - unhandled = append(unhandled, datum) } continue } - iData = append(iData, indexerData) + toIndex = append(toIndex, indexerData) } - if len(unhandled) > 0 { - for _, indexerData := range iData { - unhandled = append(unhandled, indexerData) + if err := indexer.Index(toIndex); err != nil { + log.Error("Error whilst indexing: %v Error: %v", toIndex, err) + if !indexer.Ping() { + log.Error("Issue indexer handler: indexer is unavailable when indexing") + unhandled = append(unhandled, toIndex...) } - return unhandled } - if err := indexer.Index(iData); err != nil { - log.Error("Error whilst indexing: %v Error: %v", iData, err) - if indexer.Ping() { - return nil - } - // Add back to queue - for _, indexerData := range iData { - unhandled = append(unhandled, indexerData) - } - return unhandled - } - return nil + return unhandled } - issueIndexerQueue = queue.CreateQueue("issue_indexer", handler, &IndexerData{}) + issueIndexerQueue = queue.CreateSimpleQueue("issue_indexer", handler) if issueIndexerQueue == nil { log.Fatal("Unable to create issue indexer queue") } default: - issueIndexerQueue = &queue.DummyQueue{} + issueIndexerQueue = queue.CreateSimpleQueue[*IndexerData]("issue_indexer", nil) } // Create the Indexer @@ -240,18 +221,6 @@ func InitIssueIndexer(syncReindex bool) { log.Fatal("Unknown issue indexer type: %s", setting.Indexer.IssueType) } - if queue, ok := issueIndexerQueue.(queue.Pausable); ok { - holder.get().SetAvailabilityChangeCallback(func(available bool) { - if !available { - log.Info("Issue index queue paused") - queue.Pause() - } else { - log.Info("Issue index queue resumed") - queue.Resume() - } - }) - } - // Start processing the queue go graceful.GetManager().RunWithShutdownFns(issueIndexerQueue.Run) @@ -285,9 +254,7 @@ func InitIssueIndexer(syncReindex bool) { case <-graceful.GetManager().IsShutdown(): log.Warn("Shutdown occurred before issue index initialisation was complete") case <-time.After(timeout): - if shutdownable, ok := issueIndexerQueue.(queue.Shutdownable); ok { - shutdownable.Terminate() - } + issueIndexerQueue.ShutdownWait(5 * time.Second) log.Fatal("Issue Indexer Initialization timed-out after: %v", timeout) } }() diff --git a/modules/indexer/issues/meilisearch.go b/modules/indexer/issues/meilisearch.go index 319dc3e30..990bc57a0 100644 --- a/modules/indexer/issues/meilisearch.go +++ b/modules/indexer/issues/meilisearch.go @@ -17,12 +17,11 @@ var _ Indexer = &MeilisearchIndexer{} // MeilisearchIndexer implements Indexer interface type MeilisearchIndexer struct { - client *meilisearch.Client - indexerName string - available bool - availabilityCallback func(bool) - stopTimer chan struct{} - lock sync.RWMutex + client *meilisearch.Client + indexerName string + available bool + stopTimer chan struct{} + lock sync.RWMutex } // MeilisearchIndexer creates a new meilisearch indexer @@ -73,13 +72,6 @@ func (b *MeilisearchIndexer) Init() (bool, error) { return false, b.checkError(err) } -// SetAvailabilityChangeCallback sets callback that will be triggered when availability changes -func (b *MeilisearchIndexer) SetAvailabilityChangeCallback(callback func(bool)) { - b.lock.Lock() - defer b.lock.Unlock() - b.availabilityCallback = callback -} - // Ping checks if meilisearch is available func (b *MeilisearchIndexer) Ping() bool { b.lock.RLock() @@ -178,8 +170,4 @@ func (b *MeilisearchIndexer) setAvailability(available bool) { } b.available = available - if b.availabilityCallback != nil { - // Call the callback from within the lock to ensure that the ordering remains correct - b.availabilityCallback(b.available) - } } diff --git a/modules/indexer/stats/indexer_test.go b/modules/indexer/stats/indexer_test.go index 8d9b4e36d..be9c6659f 100644 --- a/modules/indexer/stats/indexer_test.go +++ b/modules/indexer/stats/indexer_test.go @@ -41,7 +41,7 @@ func TestRepoStatsIndex(t *testing.T) { err = UpdateRepoIndexer(repo) assert.NoError(t, err) - queue.GetManager().FlushAll(context.Background(), 5*time.Second) + assert.NoError(t, queue.GetManager().FlushAll(context.Background(), 5*time.Second)) status, err := repo_model.GetIndexerStatus(db.DefaultContext, repo, repo_model.RepoIndexerTypeStats) assert.NoError(t, err) diff --git a/modules/indexer/stats/queue.go b/modules/indexer/stats/queue.go index a57338e07..46438925e 100644 --- a/modules/indexer/stats/queue.go +++ b/modules/indexer/stats/queue.go @@ -14,12 +14,11 @@ import ( ) // statsQueue represents a queue to handle repository stats updates -var statsQueue queue.UniqueQueue +var statsQueue *queue.WorkerPoolQueue[int64] // handle passed PR IDs and test the PRs -func handle(data ...queue.Data) []queue.Data { - for _, datum := range data { - opts := datum.(int64) +func handler(items ...int64) []int64 { + for _, opts := range items { if err := indexer.Index(opts); err != nil { if !setting.IsInTesting { log.Error("stats queue indexer.Index(%d) failed: %v", opts, err) @@ -30,7 +29,7 @@ func handle(data ...queue.Data) []queue.Data { } func initStatsQueue() error { - statsQueue = queue.CreateUniqueQueue("repo_stats_update", handle, int64(0)) + statsQueue = queue.CreateUniqueQueue("repo_stats_update", handler) if statsQueue == nil { return fmt.Errorf("Unable to create repo_stats_update Queue") } diff --git a/modules/mirror/mirror.go b/modules/mirror/mirror.go index 37b4c2ac9..73e591adb 100644 --- a/modules/mirror/mirror.go +++ b/modules/mirror/mirror.go @@ -10,7 +10,7 @@ import ( "code.gitea.io/gitea/modules/setting" ) -var mirrorQueue queue.UniqueQueue +var mirrorQueue *queue.WorkerPoolQueue[*SyncRequest] // SyncType type of sync request type SyncType int @@ -29,11 +29,11 @@ type SyncRequest struct { } // StartSyncMirrors starts a go routine to sync the mirrors -func StartSyncMirrors(queueHandle func(data ...queue.Data) []queue.Data) { +func StartSyncMirrors(queueHandle func(data ...*SyncRequest) []*SyncRequest) { if !setting.Mirror.Enabled { return } - mirrorQueue = queue.CreateUniqueQueue("mirror", queueHandle, new(SyncRequest)) + mirrorQueue = queue.CreateUniqueQueue("mirror", queueHandle) go graceful.GetManager().RunWithShutdownFns(mirrorQueue.Run) } diff --git a/modules/notification/ui/ui.go b/modules/notification/ui/ui.go index 73ea92274..a4576b679 100644 --- a/modules/notification/ui/ui.go +++ b/modules/notification/ui/ui.go @@ -21,7 +21,7 @@ import ( type ( notificationService struct { base.NullNotifier - issueQueue queue.Queue + issueQueue *queue.WorkerPoolQueue[issueNotificationOpts] } issueNotificationOpts struct { @@ -37,13 +37,12 @@ var _ base.Notifier = ¬ificationService{} // NewNotifier create a new notificationService notifier func NewNotifier() base.Notifier { ns := ¬ificationService{} - ns.issueQueue = queue.CreateQueue("notification-service", ns.handle, issueNotificationOpts{}) + ns.issueQueue = queue.CreateSimpleQueue("notification-service", handler) return ns } -func (ns *notificationService) handle(data ...queue.Data) []queue.Data { - for _, datum := range data { - opts := datum.(issueNotificationOpts) +func handler(items ...issueNotificationOpts) []issueNotificationOpts { + for _, opts := range items { if err := activities_model.CreateOrUpdateIssueNotifications(opts.IssueID, opts.CommentID, opts.NotificationAuthorID, opts.ReceiverID); err != nil { log.Error("Was unable to create issue notification: %v", err) } @@ -52,7 +51,7 @@ func (ns *notificationService) handle(data ...queue.Data) []queue.Data { } func (ns *notificationService) Run() { - graceful.GetManager().RunWithShutdownFns(ns.issueQueue.Run) + go graceful.GetManager().RunWithShutdownFns(ns.issueQueue.Run) } func (ns *notificationService) NotifyCreateIssueComment(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, diff --git a/modules/queue/backoff.go b/modules/queue/backoff.go new file mode 100644 index 000000000..cda723356 --- /dev/null +++ b/modules/queue/backoff.go @@ -0,0 +1,63 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "time" +) + +const ( + backoffBegin = 50 * time.Millisecond + backoffUpper = 2 * time.Second +) + +type ( + backoffFuncRetErr[T any] func() (retry bool, ret T, err error) + backoffFuncErr func() (retry bool, err error) +) + +func backoffRetErr[T any](ctx context.Context, begin, upper time.Duration, end <-chan time.Time, fn backoffFuncRetErr[T]) (ret T, err error) { + d := begin + for { + // check whether the context has been cancelled or has reached the deadline, return early + select { + case <-ctx.Done(): + return ret, ctx.Err() + case <-end: + return ret, context.DeadlineExceeded + default: + } + + // call the target function + retry, ret, err := fn() + if err != nil { + return ret, err + } + if !retry { + return ret, nil + } + + // wait for a while before retrying, and also respect the context & deadline + select { + case <-ctx.Done(): + return ret, ctx.Err() + case <-time.After(d): + d *= 2 + if d > upper { + d = upper + } + case <-end: + return ret, context.DeadlineExceeded + } + } +} + +func backoffErr(ctx context.Context, begin, upper time.Duration, end <-chan time.Time, fn backoffFuncErr) error { + _, err := backoffRetErr(ctx, begin, upper, end, func() (retry bool, ret any, err error) { + retry, err = fn() + return retry, nil, err + }) + return err +} diff --git a/modules/queue/base.go b/modules/queue/base.go new file mode 100644 index 000000000..102e79e54 --- /dev/null +++ b/modules/queue/base.go @@ -0,0 +1,42 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "time" +) + +var pushBlockTime = 5 * time.Second + +type baseQueue interface { + PushItem(ctx context.Context, data []byte) error + PopItem(ctx context.Context) ([]byte, error) + HasItem(ctx context.Context, data []byte) (bool, error) + Len(ctx context.Context) (int, error) + Close() error + RemoveAll(ctx context.Context) error +} + +func popItemByChan(ctx context.Context, popItemFn func(ctx context.Context) ([]byte, error)) (chanItem chan []byte, chanErr chan error) { + chanItem = make(chan []byte) + chanErr = make(chan error) + go func() { + for { + it, err := popItemFn(ctx) + if err != nil { + close(chanItem) + chanErr <- err + return + } + if it == nil { + close(chanItem) + close(chanErr) + return + } + chanItem <- it + } + }() + return chanItem, chanErr +} diff --git a/modules/queue/base_channel.go b/modules/queue/base_channel.go new file mode 100644 index 000000000..27055faf4 --- /dev/null +++ b/modules/queue/base_channel.go @@ -0,0 +1,123 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "errors" + "sync" + "time" + + "code.gitea.io/gitea/modules/container" +) + +var errChannelClosed = errors.New("channel is closed") + +type baseChannel struct { + c chan []byte + set container.Set[string] + mu sync.Mutex + + isUnique bool +} + +var _ baseQueue = (*baseChannel)(nil) + +func newBaseChannelGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) { + q := &baseChannel{c: make(chan []byte, cfg.Length), isUnique: unique} + if unique { + q.set = container.Set[string]{} + } + return q, nil +} + +func newBaseChannelSimple(cfg *BaseConfig) (baseQueue, error) { + return newBaseChannelGeneric(cfg, false) +} + +func newBaseChannelUnique(cfg *BaseConfig) (baseQueue, error) { + return newBaseChannelGeneric(cfg, true) +} + +func (q *baseChannel) PushItem(ctx context.Context, data []byte) error { + if q.c == nil { + return errChannelClosed + } + + if q.isUnique { + q.mu.Lock() + has := q.set.Contains(string(data)) + q.mu.Unlock() + if has { + return ErrAlreadyInQueue + } + } + + select { + case q.c <- data: + if q.isUnique { + q.mu.Lock() + q.set.Add(string(data)) + q.mu.Unlock() + } + return nil + case <-time.After(pushBlockTime): + return context.DeadlineExceeded + case <-ctx.Done(): + return ctx.Err() + } +} + +func (q *baseChannel) PopItem(ctx context.Context) ([]byte, error) { + select { + case data, ok := <-q.c: + if !ok { + return nil, errChannelClosed + } + q.mu.Lock() + q.set.Remove(string(data)) + q.mu.Unlock() + return data, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (q *baseChannel) HasItem(ctx context.Context, data []byte) (bool, error) { + q.mu.Lock() + defer q.mu.Unlock() + + return q.set.Contains(string(data)), nil +} + +func (q *baseChannel) Len(ctx context.Context) (int, error) { + q.mu.Lock() + defer q.mu.Unlock() + + if q.c == nil { + return 0, errChannelClosed + } + + return len(q.c), nil +} + +func (q *baseChannel) Close() error { + q.mu.Lock() + defer q.mu.Unlock() + + close(q.c) + q.set = container.Set[string]{} + + return nil +} + +func (q *baseChannel) RemoveAll(ctx context.Context) error { + q.mu.Lock() + defer q.mu.Unlock() + + for q.c != nil && len(q.c) > 0 { + <-q.c + } + return nil +} diff --git a/modules/queue/base_channel_test.go b/modules/queue/base_channel_test.go new file mode 100644 index 000000000..5d0a2ed0a --- /dev/null +++ b/modules/queue/base_channel_test.go @@ -0,0 +1,11 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import "testing" + +func TestBaseChannel(t *testing.T) { + testQueueBasic(t, newBaseChannelSimple, &BaseConfig{ManagedName: "baseChannel", Length: 10}, false) + testQueueBasic(t, newBaseChannelUnique, &BaseConfig{ManagedName: "baseChannel", Length: 10}, true) +} diff --git a/modules/queue/base_dummy.go b/modules/queue/base_dummy.go new file mode 100644 index 000000000..7503568a0 --- /dev/null +++ b/modules/queue/base_dummy.go @@ -0,0 +1,38 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import "context" + +type baseDummy struct{} + +var _ baseQueue = (*baseDummy)(nil) + +func newBaseDummy(cfg *BaseConfig, unique bool) (baseQueue, error) { + return &baseDummy{}, nil +} + +func (q *baseDummy) PushItem(ctx context.Context, data []byte) error { + return nil +} + +func (q *baseDummy) PopItem(ctx context.Context) ([]byte, error) { + return nil, nil +} + +func (q *baseDummy) Len(ctx context.Context) (int, error) { + return 0, nil +} + +func (q *baseDummy) HasItem(ctx context.Context, data []byte) (bool, error) { + return false, nil +} + +func (q *baseDummy) Close() error { + return nil +} + +func (q *baseDummy) RemoveAll(ctx context.Context) error { + return nil +} diff --git a/modules/queue/base_levelqueue.go b/modules/queue/base_levelqueue.go new file mode 100644 index 000000000..afde50211 --- /dev/null +++ b/modules/queue/base_levelqueue.go @@ -0,0 +1,72 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + + "code.gitea.io/gitea/modules/nosql" + + "gitea.com/lunny/levelqueue" +) + +type baseLevelQueue struct { + internal *levelqueue.Queue + conn string + cfg *BaseConfig +} + +var _ baseQueue = (*baseLevelQueue)(nil) + +func newBaseLevelQueueGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) { + if unique { + return newBaseLevelQueueUnique(cfg) + } + return newBaseLevelQueueSimple(cfg) +} + +func newBaseLevelQueueSimple(cfg *BaseConfig) (baseQueue, error) { + conn, db, err := prepareLevelDB(cfg) + if err != nil { + return nil, err + } + q := &baseLevelQueue{conn: conn, cfg: cfg} + q.internal, err = levelqueue.NewQueue(db, []byte(cfg.QueueFullName), false) + if err != nil { + return nil, err + } + + return q, nil +} + +func (q *baseLevelQueue) PushItem(ctx context.Context, data []byte) error { + return baseLevelQueueCommon(q.cfg, q.internal, nil).PushItem(ctx, data) +} + +func (q *baseLevelQueue) PopItem(ctx context.Context) ([]byte, error) { + return baseLevelQueueCommon(q.cfg, q.internal, nil).PopItem(ctx) +} + +func (q *baseLevelQueue) HasItem(ctx context.Context, data []byte) (bool, error) { + return false, nil +} + +func (q *baseLevelQueue) Len(ctx context.Context) (int, error) { + return int(q.internal.Len()), nil +} + +func (q *baseLevelQueue) Close() error { + err := q.internal.Close() + _ = nosql.GetManager().CloseLevelDB(q.conn) + return err +} + +func (q *baseLevelQueue) RemoveAll(ctx context.Context) error { + for q.internal.Len() > 0 { + if _, err := q.internal.LPop(); err != nil { + return err + } + } + return nil +} diff --git a/modules/queue/base_levelqueue_common.go b/modules/queue/base_levelqueue_common.go new file mode 100644 index 000000000..409a96551 --- /dev/null +++ b/modules/queue/base_levelqueue_common.go @@ -0,0 +1,92 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "fmt" + "path/filepath" + "strings" + "sync" + "time" + + "code.gitea.io/gitea/modules/nosql" + + "gitea.com/lunny/levelqueue" + "github.com/syndtr/goleveldb/leveldb" +) + +type baseLevelQueuePushPoper interface { + RPush(data []byte) error + LPop() ([]byte, error) + Len() int64 +} + +type baseLevelQueueCommonImpl struct { + length int + internal baseLevelQueuePushPoper + mu *sync.Mutex +} + +func (q *baseLevelQueueCommonImpl) PushItem(ctx context.Context, data []byte) error { + return backoffErr(ctx, backoffBegin, backoffUpper, time.After(pushBlockTime), func() (retry bool, err error) { + if q.mu != nil { + q.mu.Lock() + defer q.mu.Unlock() + } + + cnt := int(q.internal.Len()) + if cnt >= q.length { + return true, nil + } + retry, err = false, q.internal.RPush(data) + if err == levelqueue.ErrAlreadyInQueue { + err = ErrAlreadyInQueue + } + return retry, err + }) +} + +func (q *baseLevelQueueCommonImpl) PopItem(ctx context.Context) ([]byte, error) { + return backoffRetErr(ctx, backoffBegin, backoffUpper, infiniteTimerC, func() (retry bool, data []byte, err error) { + if q.mu != nil { + q.mu.Lock() + defer q.mu.Unlock() + } + + data, err = q.internal.LPop() + if err == levelqueue.ErrNotFound { + return true, nil, nil + } + if err != nil { + return false, nil, err + } + return false, data, nil + }) +} + +func baseLevelQueueCommon(cfg *BaseConfig, internal baseLevelQueuePushPoper, mu *sync.Mutex) *baseLevelQueueCommonImpl { + return &baseLevelQueueCommonImpl{length: cfg.Length, internal: internal} +} + +func prepareLevelDB(cfg *BaseConfig) (conn string, db *leveldb.DB, err error) { + if cfg.ConnStr == "" { // use data dir as conn str + if !filepath.IsAbs(cfg.DataFullDir) { + return "", nil, fmt.Errorf("invalid leveldb data dir (not absolute): %q", cfg.DataFullDir) + } + conn = cfg.DataFullDir + } else { + if !strings.HasPrefix(cfg.ConnStr, "leveldb://") { + return "", nil, fmt.Errorf("invalid leveldb connection string: %q", cfg.ConnStr) + } + conn = cfg.ConnStr + } + for i := 0; i < 10; i++ { + if db, err = nosql.GetManager().GetLevelDB(conn); err == nil { + break + } + time.Sleep(1 * time.Second) + } + return conn, db, err +} diff --git a/modules/queue/base_levelqueue_test.go b/modules/queue/base_levelqueue_test.go new file mode 100644 index 000000000..712a0892c --- /dev/null +++ b/modules/queue/base_levelqueue_test.go @@ -0,0 +1,23 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "testing" + + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func TestBaseLevelDB(t *testing.T) { + _, err := newBaseLevelQueueGeneric(&BaseConfig{ConnStr: "redis://"}, false) + assert.ErrorContains(t, err, "invalid leveldb connection string") + + _, err = newBaseLevelQueueGeneric(&BaseConfig{DataFullDir: "relative"}, false) + assert.ErrorContains(t, err, "invalid leveldb data dir") + + testQueueBasic(t, newBaseLevelQueueSimple, toBaseConfig("baseLevelQueue", setting.QueueSettings{Datadir: t.TempDir() + "/queue-test", Length: 10}), false) + testQueueBasic(t, newBaseLevelQueueUnique, toBaseConfig("baseLevelQueueUnique", setting.QueueSettings{ConnStr: "leveldb://" + t.TempDir() + "/queue-test", Length: 10}), true) +} diff --git a/modules/queue/base_levelqueue_unique.go b/modules/queue/base_levelqueue_unique.go new file mode 100644 index 000000000..754622163 --- /dev/null +++ b/modules/queue/base_levelqueue_unique.go @@ -0,0 +1,93 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "sync" + "unsafe" + + "code.gitea.io/gitea/modules/nosql" + + "gitea.com/lunny/levelqueue" + "github.com/syndtr/goleveldb/leveldb" +) + +type baseLevelQueueUnique struct { + internal *levelqueue.UniqueQueue + conn string + cfg *BaseConfig + + mu sync.Mutex // the levelqueue.UniqueQueue is not thread-safe, there is no mutex protecting the underlying queue&set together +} + +var _ baseQueue = (*baseLevelQueueUnique)(nil) + +func newBaseLevelQueueUnique(cfg *BaseConfig) (baseQueue, error) { + conn, db, err := prepareLevelDB(cfg) + if err != nil { + return nil, err + } + q := &baseLevelQueueUnique{conn: conn, cfg: cfg} + q.internal, err = levelqueue.NewUniqueQueue(db, []byte(cfg.QueueFullName), []byte(cfg.SetFullName), false) + if err != nil { + return nil, err + } + + return q, nil +} + +func (q *baseLevelQueueUnique) PushItem(ctx context.Context, data []byte) error { + return baseLevelQueueCommon(q.cfg, q.internal, &q.mu).PushItem(ctx, data) +} + +func (q *baseLevelQueueUnique) PopItem(ctx context.Context) ([]byte, error) { + return baseLevelQueueCommon(q.cfg, q.internal, &q.mu).PopItem(ctx) +} + +func (q *baseLevelQueueUnique) HasItem(ctx context.Context, data []byte) (bool, error) { + q.mu.Lock() + defer q.mu.Unlock() + return q.internal.Has(data) +} + +func (q *baseLevelQueueUnique) Len(ctx context.Context) (int, error) { + q.mu.Lock() + defer q.mu.Unlock() + return int(q.internal.Len()), nil +} + +func (q *baseLevelQueueUnique) Close() error { + q.mu.Lock() + defer q.mu.Unlock() + err := q.internal.Close() + _ = nosql.GetManager().CloseLevelDB(q.conn) + return err +} + +func (q *baseLevelQueueUnique) RemoveAll(ctx context.Context) error { + q.mu.Lock() + defer q.mu.Unlock() + + type levelUniqueQueue struct { + q *levelqueue.Queue + set *levelqueue.Set + db *leveldb.DB + } + lq := (*levelUniqueQueue)(unsafe.Pointer(q.internal)) + + members, err := lq.set.Members() + if err != nil { + return err // seriously corrupted + } + for _, v := range members { + _, _ = lq.set.Remove(v) + } + for lq.q.Len() > 0 { + if _, err = lq.q.LPop(); err != nil { + return err + } + } + return nil +} diff --git a/modules/queue/base_redis.go b/modules/queue/base_redis.go new file mode 100644 index 000000000..a294077cc --- /dev/null +++ b/modules/queue/base_redis.go @@ -0,0 +1,135 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "sync" + "time" + + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/nosql" + + "github.com/redis/go-redis/v9" +) + +type baseRedis struct { + client redis.UniversalClient + isUnique bool + cfg *BaseConfig + + mu sync.Mutex // the old implementation is not thread-safe, the queue operation and set operation should be protected together +} + +var _ baseQueue = (*baseRedis)(nil) + +func newBaseRedisGeneric(cfg *BaseConfig, unique bool) (baseQueue, error) { + client := nosql.GetManager().GetRedisClient(cfg.ConnStr) + + var err error + for i := 0; i < 10; i++ { + err = client.Ping(graceful.GetManager().ShutdownContext()).Err() + if err == nil { + break + } + log.Warn("Redis is not ready, waiting for 1 second to retry: %v", err) + time.Sleep(time.Second) + } + if err != nil { + return nil, err + } + + return &baseRedis{cfg: cfg, client: client, isUnique: unique}, nil +} + +func newBaseRedisSimple(cfg *BaseConfig) (baseQueue, error) { + return newBaseRedisGeneric(cfg, false) +} + +func newBaseRedisUnique(cfg *BaseConfig) (baseQueue, error) { + return newBaseRedisGeneric(cfg, true) +} + +func (q *baseRedis) PushItem(ctx context.Context, data []byte) error { + return backoffErr(ctx, backoffBegin, backoffUpper, time.After(pushBlockTime), func() (retry bool, err error) { + q.mu.Lock() + defer q.mu.Unlock() + + cnt, err := q.client.LLen(ctx, q.cfg.QueueFullName).Result() + if err != nil { + return false, err + } + if int(cnt) >= q.cfg.Length { + return true, nil + } + + if q.isUnique { + added, err := q.client.SAdd(ctx, q.cfg.SetFullName, data).Result() + if err != nil { + return false, err + } + if added == 0 { + return false, ErrAlreadyInQueue + } + } + return false, q.client.RPush(ctx, q.cfg.QueueFullName, data).Err() + }) +} + +func (q *baseRedis) PopItem(ctx context.Context) ([]byte, error) { + return backoffRetErr(ctx, backoffBegin, backoffUpper, infiniteTimerC, func() (retry bool, data []byte, err error) { + q.mu.Lock() + defer q.mu.Unlock() + + data, err = q.client.LPop(ctx, q.cfg.QueueFullName).Bytes() + if err == redis.Nil { + return true, nil, nil + } + if err != nil { + return true, nil, nil + } + if q.isUnique { + // the data has been popped, even if there is any error we can't do anything + _ = q.client.SRem(ctx, q.cfg.SetFullName, data).Err() + } + return false, data, err + }) +} + +func (q *baseRedis) HasItem(ctx context.Context, data []byte) (bool, error) { + q.mu.Lock() + defer q.mu.Unlock() + if !q.isUnique { + return false, nil + } + return q.client.SIsMember(ctx, q.cfg.SetFullName, data).Result() +} + +func (q *baseRedis) Len(ctx context.Context) (int, error) { + q.mu.Lock() + defer q.mu.Unlock() + cnt, err := q.client.LLen(ctx, q.cfg.QueueFullName).Result() + return int(cnt), err +} + +func (q *baseRedis) Close() error { + q.mu.Lock() + defer q.mu.Unlock() + return q.client.Close() +} + +func (q *baseRedis) RemoveAll(ctx context.Context) error { + q.mu.Lock() + defer q.mu.Unlock() + c1 := q.client.Del(ctx, q.cfg.QueueFullName) + c2 := q.client.Del(ctx, q.cfg.SetFullName) + if c1.Err() != nil { + return c1.Err() + } + if c2.Err() != nil { + return c2.Err() + } + return nil // actually, checking errors doesn't make sense here because the state could be out-of-sync +} diff --git a/modules/queue/base_redis_test.go b/modules/queue/base_redis_test.go new file mode 100644 index 000000000..3d49e8d98 --- /dev/null +++ b/modules/queue/base_redis_test.go @@ -0,0 +1,71 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "os" + "os/exec" + "testing" + "time" + + "code.gitea.io/gitea/modules/nosql" + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func waitRedisReady(conn string, dur time.Duration) (ready bool) { + ctxTimed, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + for t := time.Now(); ; time.Sleep(50 * time.Millisecond) { + ret := nosql.GetManager().GetRedisClient(conn).Ping(ctxTimed) + if ret.Err() == nil { + return true + } + if time.Since(t) > dur { + return false + } + } +} + +func redisServerCmd(t *testing.T) *exec.Cmd { + redisServerProg, err := exec.LookPath("redis-server") + if err != nil { + return nil + } + c := &exec.Cmd{ + Path: redisServerProg, + Args: []string{redisServerProg, "--bind", "127.0.0.1", "--port", "6379"}, + Dir: t.TempDir(), + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + } + return c +} + +func TestBaseRedis(t *testing.T) { + var redisServer *exec.Cmd + defer func() { + if redisServer != nil { + _ = redisServer.Process.Signal(os.Interrupt) + _ = redisServer.Wait() + } + }() + if !waitRedisReady("redis://127.0.0.1:6379/0", 0) { + redisServer = redisServerCmd(t) + if redisServer == nil && os.Getenv("CI") != "" { + t.Skip("redis-server not found") + return + } + assert.NoError(t, redisServer.Start()) + if !assert.True(t, waitRedisReady("redis://127.0.0.1:6379/0", 5*time.Second), "start redis-server") { + return + } + } + + testQueueBasic(t, newBaseRedisSimple, toBaseConfig("baseRedis", setting.QueueSettings{Length: 10}), false) + testQueueBasic(t, newBaseRedisUnique, toBaseConfig("baseRedisUnique", setting.QueueSettings{Length: 10}), true) +} diff --git a/modules/queue/base_test.go b/modules/queue/base_test.go new file mode 100644 index 000000000..c5bf526ae --- /dev/null +++ b/modules/queue/base_test.go @@ -0,0 +1,140 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func testQueueBasic(t *testing.T, newFn func(cfg *BaseConfig) (baseQueue, error), cfg *BaseConfig, isUnique bool) { + t.Run(fmt.Sprintf("testQueueBasic-%s-unique:%v", cfg.ManagedName, isUnique), func(t *testing.T) { + q, err := newFn(cfg) + assert.NoError(t, err) + + ctx := context.Background() + _ = q.RemoveAll(ctx) + cnt, err := q.Len(ctx) + assert.NoError(t, err) + assert.EqualValues(t, 0, cnt) + + // push the first item + err = q.PushItem(ctx, []byte("foo")) + assert.NoError(t, err) + + cnt, err = q.Len(ctx) + assert.NoError(t, err) + assert.EqualValues(t, 1, cnt) + + // push a duplicate item + err = q.PushItem(ctx, []byte("foo")) + if !isUnique { + assert.NoError(t, err) + } else { + assert.ErrorIs(t, err, ErrAlreadyInQueue) + } + + // check the duplicate item + cnt, err = q.Len(ctx) + assert.NoError(t, err) + has, err := q.HasItem(ctx, []byte("foo")) + assert.NoError(t, err) + if !isUnique { + assert.EqualValues(t, 2, cnt) + assert.EqualValues(t, false, has) // non-unique queues don't check for duplicates + } else { + assert.EqualValues(t, 1, cnt) + assert.EqualValues(t, true, has) + } + + // push another item + err = q.PushItem(ctx, []byte("bar")) + assert.NoError(t, err) + + // pop the first item (and the duplicate if non-unique) + it, err := q.PopItem(ctx) + assert.NoError(t, err) + assert.EqualValues(t, "foo", string(it)) + + if !isUnique { + it, err = q.PopItem(ctx) + assert.NoError(t, err) + assert.EqualValues(t, "foo", string(it)) + } + + // pop another item + it, err = q.PopItem(ctx) + assert.NoError(t, err) + assert.EqualValues(t, "bar", string(it)) + + // pop an empty queue (timeout, cancel) + ctxTimed, cancel := context.WithTimeout(ctx, 10*time.Millisecond) + it, err = q.PopItem(ctxTimed) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.Nil(t, it) + cancel() + + ctxTimed, cancel = context.WithTimeout(ctx, 10*time.Millisecond) + cancel() + it, err = q.PopItem(ctxTimed) + assert.ErrorIs(t, err, context.Canceled) + assert.Nil(t, it) + + // test blocking push if queue is full + for i := 0; i < cfg.Length; i++ { + err = q.PushItem(ctx, []byte(fmt.Sprintf("item-%d", i))) + assert.NoError(t, err) + } + ctxTimed, cancel = context.WithTimeout(ctx, 10*time.Millisecond) + err = q.PushItem(ctxTimed, []byte("item-full")) + assert.ErrorIs(t, err, context.DeadlineExceeded) + cancel() + + // test blocking push if queue is full (with custom pushBlockTime) + oldPushBlockTime := pushBlockTime + timeStart := time.Now() + pushBlockTime = 30 * time.Millisecond + err = q.PushItem(ctx, []byte("item-full")) + assert.ErrorIs(t, err, context.DeadlineExceeded) + assert.True(t, time.Since(timeStart) >= pushBlockTime*2/3) + pushBlockTime = oldPushBlockTime + + // remove all + cnt, err = q.Len(ctx) + assert.NoError(t, err) + assert.EqualValues(t, cfg.Length, cnt) + + _ = q.RemoveAll(ctx) + + cnt, err = q.Len(ctx) + assert.NoError(t, err) + assert.EqualValues(t, 0, cnt) + }) +} + +func TestBaseDummy(t *testing.T) { + q, err := newBaseDummy(&BaseConfig{}, true) + assert.NoError(t, err) + + ctx := context.Background() + assert.NoError(t, q.PushItem(ctx, []byte("foo"))) + + cnt, err := q.Len(ctx) + assert.NoError(t, err) + assert.EqualValues(t, 0, cnt) + + has, err := q.HasItem(ctx, []byte("foo")) + assert.NoError(t, err) + assert.False(t, has) + + it, err := q.PopItem(ctx) + assert.NoError(t, err) + assert.Nil(t, it) + + assert.NoError(t, q.RemoveAll(ctx)) +} diff --git a/modules/queue/bytefifo.go b/modules/queue/bytefifo.go deleted file mode 100644 index c33b79426..000000000 --- a/modules/queue/bytefifo.go +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import "context" - -// ByteFIFO defines a FIFO that takes a byte array -type ByteFIFO interface { - // Len returns the length of the fifo - Len(ctx context.Context) int64 - // PushFunc pushes data to the end of the fifo and calls the callback if it is added - PushFunc(ctx context.Context, data []byte, fn func() error) error - // Pop pops data from the start of the fifo - Pop(ctx context.Context) ([]byte, error) - // Close this fifo - Close() error - // PushBack pushes data back to the top of the fifo - PushBack(ctx context.Context, data []byte) error -} - -// UniqueByteFIFO defines a FIFO that Uniques its contents -type UniqueByteFIFO interface { - ByteFIFO - // Has returns whether the fifo contains this data - Has(ctx context.Context, data []byte) (bool, error) -} - -var _ ByteFIFO = &DummyByteFIFO{} - -// DummyByteFIFO represents a dummy fifo -type DummyByteFIFO struct{} - -// PushFunc returns nil -func (*DummyByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { - return nil -} - -// Pop returns nil -func (*DummyByteFIFO) Pop(ctx context.Context) ([]byte, error) { - return []byte{}, nil -} - -// Close returns nil -func (*DummyByteFIFO) Close() error { - return nil -} - -// Len is always 0 -func (*DummyByteFIFO) Len(ctx context.Context) int64 { - return 0 -} - -// PushBack pushes data back to the top of the fifo -func (*DummyByteFIFO) PushBack(ctx context.Context, data []byte) error { - return nil -} - -var _ UniqueByteFIFO = &DummyUniqueByteFIFO{} - -// DummyUniqueByteFIFO represents a dummy unique fifo -type DummyUniqueByteFIFO struct { - DummyByteFIFO -} - -// Has always returns false -func (*DummyUniqueByteFIFO) Has(ctx context.Context, data []byte) (bool, error) { - return false, nil -} diff --git a/modules/queue/config.go b/modules/queue/config.go new file mode 100644 index 000000000..c5bc16b6f --- /dev/null +++ b/modules/queue/config.go @@ -0,0 +1,36 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "code.gitea.io/gitea/modules/setting" +) + +type BaseConfig struct { + ManagedName string + DataFullDir string // the caller must prepare an absolute path + + ConnStr string + Length int + + QueueFullName, SetFullName string +} + +func toBaseConfig(managedName string, queueSetting setting.QueueSettings) *BaseConfig { + baseConfig := &BaseConfig{ + ManagedName: managedName, + DataFullDir: queueSetting.Datadir, + + ConnStr: queueSetting.ConnStr, + Length: queueSetting.Length, + } + + // queue name and set name + baseConfig.QueueFullName = managedName + queueSetting.QueueName + baseConfig.SetFullName = baseConfig.QueueFullName + queueSetting.SetName + if baseConfig.SetFullName == baseConfig.QueueFullName { + baseConfig.SetFullName += "_unique" + } + return baseConfig +} diff --git a/modules/queue/helper.go b/modules/queue/helper.go deleted file mode 100644 index c6fb9447b..000000000 --- a/modules/queue/helper.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "reflect" - - "code.gitea.io/gitea/modules/json" -) - -// Mappable represents an interface that can MapTo another interface -type Mappable interface { - MapTo(v interface{}) error -} - -// toConfig will attempt to convert a given configuration cfg into the provided exemplar type. -// -// It will tolerate the cfg being passed as a []byte or string of a json representation of the -// exemplar or the correct type of the exemplar itself -func toConfig(exemplar, cfg interface{}) (interface{}, error) { - // First of all check if we've got the same type as the exemplar - if so it's all fine. - if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) { - return cfg, nil - } - - // Now if not - does it provide a MapTo function we can try? - if mappable, ok := cfg.(Mappable); ok { - newVal := reflect.New(reflect.TypeOf(exemplar)) - if err := mappable.MapTo(newVal.Interface()); err == nil { - return newVal.Elem().Interface(), nil - } - // MapTo has failed us ... let's try the json route ... - } - - // OK we've been passed a byte array right? - configBytes, ok := cfg.([]byte) - if !ok { - // oh ... it's a string then? - var configStr string - - configStr, ok = cfg.(string) - configBytes = []byte(configStr) - } - if !ok { - // hmm ... can we marshal it to json? - var err error - configBytes, err = json.Marshal(cfg) - ok = err == nil - } - if !ok { - // no ... we've tried hard enough at this point - throw an error! - return nil, ErrInvalidConfiguration{cfg: cfg} - } - - // OK unmarshal the byte array into a new copy of the exemplar - newVal := reflect.New(reflect.TypeOf(exemplar)) - if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil { - // If we can't unmarshal it then return an error! - return nil, ErrInvalidConfiguration{cfg: cfg, err: err} - } - return newVal.Elem().Interface(), nil -} - -// unmarshalAs will attempt to unmarshal provided bytes as the provided exemplar -func unmarshalAs(bs []byte, exemplar interface{}) (data Data, err error) { - if exemplar != nil { - t := reflect.TypeOf(exemplar) - n := reflect.New(t) - ne := n.Elem() - err = json.Unmarshal(bs, ne.Addr().Interface()) - data = ne.Interface().(Data) - } else { - err = json.Unmarshal(bs, &data) - } - return data, err -} - -// assignableTo will check if provided data is assignable to the same type as the exemplar -// if the provided exemplar is nil then it will always return true -func assignableTo(data Data, exemplar interface{}) bool { - if exemplar == nil { - return true - } - - // Assert data is of same type as exemplar - t := reflect.TypeOf(data) - exemplarType := reflect.TypeOf(exemplar) - - return t.AssignableTo(exemplarType) && data != nil -} diff --git a/modules/queue/manager.go b/modules/queue/manager.go index 6975e0290..03dbc72da 100644 --- a/modules/queue/manager.go +++ b/modules/queue/manager.go @@ -5,457 +5,106 @@ package queue import ( "context" - "fmt" - "reflect" - "sort" - "strings" "sync" "time" - "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" ) +// Manager is a manager for the queues created by "CreateXxxQueue" functions, these queues are called "managed queues". +type Manager struct { + mu sync.Mutex + + qidCounter int64 + Queues map[int64]ManagedWorkerPoolQueue +} + +type ManagedWorkerPoolQueue interface { + GetName() string + GetType() string + GetItemTypeName() string + GetWorkerNumber() int + GetWorkerActiveNumber() int + GetWorkerMaxNumber() int + SetWorkerMaxNumber(num int) + GetQueueItemNumber() int + + // FlushWithContext tries to make the handler process all items in the queue synchronously. + // It is for testing purpose only. It's not designed to be used in a cluster. + FlushWithContext(ctx context.Context, timeout time.Duration) error +} + var manager *Manager -// Manager is a queue manager -type Manager struct { - mutex sync.Mutex - - counter int64 - Queues map[int64]*ManagedQueue -} - -// ManagedQueue represents a working queue with a Pool of workers. -// -// Although a ManagedQueue should really represent a Queue this does not -// necessarily have to be the case. This could be used to describe any queue.WorkerPool. -type ManagedQueue struct { - mutex sync.Mutex - QID int64 - Type Type - Name string - Configuration interface{} - ExemplarType string - Managed interface{} - counter int64 - PoolWorkers map[int64]*PoolWorkers -} - -// Flushable represents a pool or queue that is flushable -type Flushable interface { - // Flush will add a flush worker to the pool - the worker should be autoregistered with the manager - Flush(time.Duration) error - // FlushWithContext is very similar to Flush - // NB: The worker will not be registered with the manager. - FlushWithContext(ctx context.Context) error - // IsEmpty will return if the managed pool is empty and has no work - IsEmpty() bool -} - -// Pausable represents a pool or queue that is Pausable -type Pausable interface { - // IsPaused will return if the pool or queue is paused - IsPaused() bool - // Pause will pause the pool or queue - Pause() - // Resume will resume the pool or queue - Resume() - // IsPausedIsResumed will return a bool indicating if the pool or queue is paused and a channel that will be closed when it is resumed - IsPausedIsResumed() (paused, resumed <-chan struct{}) -} - -// ManagedPool is a simple interface to get certain details from a worker pool -type ManagedPool interface { - // AddWorkers adds a number of worker as group to the pool with the provided timeout. A CancelFunc is provided to cancel the group - AddWorkers(number int, timeout time.Duration) context.CancelFunc - // NumberOfWorkers returns the total number of workers in the pool - NumberOfWorkers() int - // MaxNumberOfWorkers returns the maximum number of workers the pool can dynamically grow to - MaxNumberOfWorkers() int - // SetMaxNumberOfWorkers sets the maximum number of workers the pool can dynamically grow to - SetMaxNumberOfWorkers(int) - // BoostTimeout returns the current timeout for worker groups created during a boost - BoostTimeout() time.Duration - // BlockTimeout returns the timeout the internal channel can block for before a boost would occur - BlockTimeout() time.Duration - // BoostWorkers sets the number of workers to be created during a boost - BoostWorkers() int - // SetPoolSettings sets the user updatable settings for the pool - SetPoolSettings(maxNumberOfWorkers, boostWorkers int, timeout time.Duration) - // NumberInQueue returns the total number of items in the pool - NumberInQueue() int64 - // Done returns a channel that will be closed when the Pool's baseCtx is closed - Done() <-chan struct{} -} - -// ManagedQueueList implements the sort.Interface -type ManagedQueueList []*ManagedQueue - -// PoolWorkers represents a group of workers working on a queue -type PoolWorkers struct { - PID int64 - Workers int - Start time.Time - Timeout time.Time - HasTimeout bool - Cancel context.CancelFunc - IsFlusher bool -} - -// PoolWorkersList implements the sort.Interface for PoolWorkers -type PoolWorkersList []*PoolWorkers - func init() { - _ = GetManager() + manager = &Manager{ + Queues: make(map[int64]ManagedWorkerPoolQueue), + } } -// GetManager returns a Manager and initializes one as singleton if there's none yet func GetManager() *Manager { - if manager == nil { - manager = &Manager{ - Queues: make(map[int64]*ManagedQueue), - } - } return manager } -// Add adds a queue to this manager -func (m *Manager) Add(managed interface{}, - t Type, - configuration, - exemplar interface{}, -) int64 { - cfg, _ := json.Marshal(configuration) - mq := &ManagedQueue{ - Type: t, - Configuration: string(cfg), - ExemplarType: reflect.TypeOf(exemplar).String(), - PoolWorkers: make(map[int64]*PoolWorkers), - Managed: managed, - } - m.mutex.Lock() - m.counter++ - mq.QID = m.counter - mq.Name = fmt.Sprintf("queue-%d", mq.QID) - if named, ok := managed.(Named); ok { - name := named.Name() - if len(name) > 0 { - mq.Name = name - } - } - m.Queues[mq.QID] = mq - m.mutex.Unlock() - log.Trace("Queue Manager registered: %s (QID: %d)", mq.Name, mq.QID) - return mq.QID +func (m *Manager) AddManagedQueue(managed ManagedWorkerPoolQueue) { + m.mu.Lock() + defer m.mu.Unlock() + m.qidCounter++ + m.Queues[m.qidCounter] = managed } -// Remove a queue from the Manager -func (m *Manager) Remove(qid int64) { - m.mutex.Lock() - delete(m.Queues, qid) - m.mutex.Unlock() - log.Trace("Queue Manager removed: QID: %d", qid) -} - -// GetManagedQueue by qid -func (m *Manager) GetManagedQueue(qid int64) *ManagedQueue { - m.mutex.Lock() - defer m.mutex.Unlock() +func (m *Manager) GetManagedQueue(qid int64) ManagedWorkerPoolQueue { + m.mu.Lock() + defer m.mu.Unlock() return m.Queues[qid] } -// FlushAll flushes all the flushable queues attached to this manager -func (m *Manager) FlushAll(baseCtx context.Context, timeout time.Duration) error { - var ctx context.Context - var cancel context.CancelFunc - start := time.Now() - end := start - hasTimeout := false - if timeout > 0 { - ctx, cancel = context.WithTimeout(baseCtx, timeout) - end = start.Add(timeout) - hasTimeout = true - } else { - ctx, cancel = context.WithCancel(baseCtx) - } - defer cancel() +func (m *Manager) ManagedQueues() map[int64]ManagedWorkerPoolQueue { + m.mu.Lock() + defer m.mu.Unlock() - for { - select { - case <-ctx.Done(): - mqs := m.ManagedQueues() - nonEmptyQueues := []string{} - for _, mq := range mqs { - if !mq.IsEmpty() { - nonEmptyQueues = append(nonEmptyQueues, mq.Name) - } - } - if len(nonEmptyQueues) > 0 { - return fmt.Errorf("flush timeout with non-empty queues: %s", strings.Join(nonEmptyQueues, ", ")) - } - return nil - default: + queues := make(map[int64]ManagedWorkerPoolQueue, len(m.Queues)) + for k, v := range m.Queues { + queues[k] = v + } + return queues +} + +// FlushAll tries to make all managed queues process all items synchronously, until timeout or the queue is empty. +// It is for testing purpose only. It's not designed to be used in a cluster. +func (m *Manager) FlushAll(ctx context.Context, timeout time.Duration) error { + var finalErr error + qs := m.ManagedQueues() + for _, q := range qs { + if err := q.FlushWithContext(ctx, timeout); err != nil { + finalErr = err // TODO: in Go 1.20: errors.Join } - mqs := m.ManagedQueues() - log.Debug("Found %d Managed Queues", len(mqs)) - wg := sync.WaitGroup{} - wg.Add(len(mqs)) - allEmpty := true - for _, mq := range mqs { - if mq.IsEmpty() { - wg.Done() - continue - } - if pausable, ok := mq.Managed.(Pausable); ok { - // no point flushing paused queues - if pausable.IsPaused() { - wg.Done() - continue - } - } - if pool, ok := mq.Managed.(ManagedPool); ok { - // No point into flushing pools when their base's ctx is already done. - select { - case <-pool.Done(): - wg.Done() - continue - default: - } - } - - allEmpty = false - if flushable, ok := mq.Managed.(Flushable); ok { - log.Debug("Flushing (flushable) queue: %s", mq.Name) - go func(q *ManagedQueue) { - localCtx, localCtxCancel := context.WithCancel(ctx) - pid := q.RegisterWorkers(1, start, hasTimeout, end, localCtxCancel, true) - err := flushable.FlushWithContext(localCtx) - if err != nil && err != ctx.Err() { - cancel() - } - q.CancelWorkers(pid) - localCtxCancel() - wg.Done() - }(mq) - } else { - log.Debug("Queue: %s is non-empty but is not flushable", mq.Name) - wg.Done() - } - } - if allEmpty { - log.Debug("All queues are empty") - break - } - // Ensure there are always at least 100ms between loops but not more if we've actually been doing some flushing - // but don't delay cancellation here. - select { - case <-ctx.Done(): - case <-time.After(100 * time.Millisecond): - } - wg.Wait() } - return nil + return finalErr } -// ManagedQueues returns the managed queues -func (m *Manager) ManagedQueues() []*ManagedQueue { - m.mutex.Lock() - mqs := make([]*ManagedQueue, 0, len(m.Queues)) - for _, mq := range m.Queues { - mqs = append(mqs, mq) +// CreateSimpleQueue creates a simple queue from global setting config provider by name +func CreateSimpleQueue[T any](name string, handler HandlerFuncT[T]) *WorkerPoolQueue[T] { + return createWorkerPoolQueue(name, setting.CfgProvider, handler, false) +} + +// CreateUniqueQueue creates a unique queue from global setting config provider by name +func CreateUniqueQueue[T any](name string, handler HandlerFuncT[T]) *WorkerPoolQueue[T] { + return createWorkerPoolQueue(name, setting.CfgProvider, handler, true) +} + +func createWorkerPoolQueue[T any](name string, cfgProvider setting.ConfigProvider, handler HandlerFuncT[T], unique bool) *WorkerPoolQueue[T] { + queueSetting, err := setting.GetQueueSettings(cfgProvider, name) + if err != nil { + log.Error("Failed to get queue settings for %q: %v", name, err) + return nil } - m.mutex.Unlock() - sort.Sort(ManagedQueueList(mqs)) - return mqs -} - -// Workers returns the poolworkers -func (q *ManagedQueue) Workers() []*PoolWorkers { - q.mutex.Lock() - workers := make([]*PoolWorkers, 0, len(q.PoolWorkers)) - for _, worker := range q.PoolWorkers { - workers = append(workers, worker) + w, err := NewWorkerPoolQueueBySetting(name, queueSetting, handler, unique) + if err != nil { + log.Error("Failed to create queue %q: %v", name, err) + return nil } - q.mutex.Unlock() - sort.Sort(PoolWorkersList(workers)) - return workers -} - -// RegisterWorkers registers workers to this queue -func (q *ManagedQueue) RegisterWorkers(number int, start time.Time, hasTimeout bool, timeout time.Time, cancel context.CancelFunc, isFlusher bool) int64 { - q.mutex.Lock() - defer q.mutex.Unlock() - q.counter++ - q.PoolWorkers[q.counter] = &PoolWorkers{ - PID: q.counter, - Workers: number, - Start: start, - Timeout: timeout, - HasTimeout: hasTimeout, - Cancel: cancel, - IsFlusher: isFlusher, - } - return q.counter -} - -// CancelWorkers cancels pooled workers with pid -func (q *ManagedQueue) CancelWorkers(pid int64) { - q.mutex.Lock() - pw, ok := q.PoolWorkers[pid] - q.mutex.Unlock() - if !ok { - return - } - pw.Cancel() -} - -// RemoveWorkers deletes pooled workers with pid -func (q *ManagedQueue) RemoveWorkers(pid int64) { - q.mutex.Lock() - pw, ok := q.PoolWorkers[pid] - delete(q.PoolWorkers, pid) - q.mutex.Unlock() - if ok && pw.Cancel != nil { - pw.Cancel() - } -} - -// AddWorkers adds workers to the queue if it has registered an add worker function -func (q *ManagedQueue) AddWorkers(number int, timeout time.Duration) context.CancelFunc { - if pool, ok := q.Managed.(ManagedPool); ok { - // the cancel will be added to the pool workers description above - return pool.AddWorkers(number, timeout) - } - return nil -} - -// Flushable returns true if the queue is flushable -func (q *ManagedQueue) Flushable() bool { - _, ok := q.Managed.(Flushable) - return ok -} - -// Flush flushes the queue with a timeout -func (q *ManagedQueue) Flush(timeout time.Duration) error { - if flushable, ok := q.Managed.(Flushable); ok { - // the cancel will be added to the pool workers description above - return flushable.Flush(timeout) - } - return nil -} - -// IsEmpty returns if the queue is empty -func (q *ManagedQueue) IsEmpty() bool { - if flushable, ok := q.Managed.(Flushable); ok { - return flushable.IsEmpty() - } - return true -} - -// Pausable returns whether the queue is Pausable -func (q *ManagedQueue) Pausable() bool { - _, ok := q.Managed.(Pausable) - return ok -} - -// Pause pauses the queue -func (q *ManagedQueue) Pause() { - if pausable, ok := q.Managed.(Pausable); ok { - pausable.Pause() - } -} - -// IsPaused reveals if the queue is paused -func (q *ManagedQueue) IsPaused() bool { - if pausable, ok := q.Managed.(Pausable); ok { - return pausable.IsPaused() - } - return false -} - -// Resume resumes the queue -func (q *ManagedQueue) Resume() { - if pausable, ok := q.Managed.(Pausable); ok { - pausable.Resume() - } -} - -// NumberOfWorkers returns the number of workers in the queue -func (q *ManagedQueue) NumberOfWorkers() int { - if pool, ok := q.Managed.(ManagedPool); ok { - return pool.NumberOfWorkers() - } - return -1 -} - -// MaxNumberOfWorkers returns the maximum number of workers for the pool -func (q *ManagedQueue) MaxNumberOfWorkers() int { - if pool, ok := q.Managed.(ManagedPool); ok { - return pool.MaxNumberOfWorkers() - } - return 0 -} - -// BoostWorkers returns the number of workers for a boost -func (q *ManagedQueue) BoostWorkers() int { - if pool, ok := q.Managed.(ManagedPool); ok { - return pool.BoostWorkers() - } - return -1 -} - -// BoostTimeout returns the timeout of the next boost -func (q *ManagedQueue) BoostTimeout() time.Duration { - if pool, ok := q.Managed.(ManagedPool); ok { - return pool.BoostTimeout() - } - return 0 -} - -// BlockTimeout returns the timeout til the next boost -func (q *ManagedQueue) BlockTimeout() time.Duration { - if pool, ok := q.Managed.(ManagedPool); ok { - return pool.BlockTimeout() - } - return 0 -} - -// SetPoolSettings sets the setable boost values -func (q *ManagedQueue) SetPoolSettings(maxNumberOfWorkers, boostWorkers int, timeout time.Duration) { - if pool, ok := q.Managed.(ManagedPool); ok { - pool.SetPoolSettings(maxNumberOfWorkers, boostWorkers, timeout) - } -} - -// NumberInQueue returns the number of items in the queue -func (q *ManagedQueue) NumberInQueue() int64 { - if pool, ok := q.Managed.(ManagedPool); ok { - return pool.NumberInQueue() - } - return -1 -} - -func (l ManagedQueueList) Len() int { - return len(l) -} - -func (l ManagedQueueList) Less(i, j int) bool { - return l[i].Name < l[j].Name -} - -func (l ManagedQueueList) Swap(i, j int) { - l[i], l[j] = l[j], l[i] -} - -func (l PoolWorkersList) Len() int { - return len(l) -} - -func (l PoolWorkersList) Less(i, j int) bool { - return l[i].Start.Before(l[j].Start) -} - -func (l PoolWorkersList) Swap(i, j int) { - l[i], l[j] = l[j], l[i] + GetManager().AddManagedQueue(w) + return w } diff --git a/modules/queue/manager_test.go b/modules/queue/manager_test.go new file mode 100644 index 000000000..50265e27b --- /dev/null +++ b/modules/queue/manager_test.go @@ -0,0 +1,124 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "path/filepath" + "testing" + + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func TestManager(t *testing.T) { + oldAppDataPath := setting.AppDataPath + setting.AppDataPath = t.TempDir() + defer func() { + setting.AppDataPath = oldAppDataPath + }() + + newQueueFromConfig := func(name, cfg string) (*WorkerPoolQueue[int], error) { + cfgProvider, err := setting.NewConfigProviderFromData(cfg) + if err != nil { + return nil, err + } + qs, err := setting.GetQueueSettings(cfgProvider, name) + if err != nil { + return nil, err + } + return NewWorkerPoolQueueBySetting(name, qs, func(s ...int) (unhandled []int) { return nil }, false) + } + + // test invalid CONN_STR + _, err := newQueueFromConfig("default", ` +[queue] +DATADIR = temp-dir +CONN_STR = redis:// +`) + assert.ErrorContains(t, err, "invalid leveldb connection string") + + // test default config + q, err := newQueueFromConfig("default", "") + assert.NoError(t, err) + assert.Equal(t, "default", q.GetName()) + assert.Equal(t, "level", q.GetType()) + assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/common"), q.baseConfig.DataFullDir) + assert.Equal(t, 100, q.baseConfig.Length) + assert.Equal(t, 20, q.batchLength) + assert.Equal(t, "", q.baseConfig.ConnStr) + assert.Equal(t, "default_queue", q.baseConfig.QueueFullName) + assert.Equal(t, "default_queue_unique", q.baseConfig.SetFullName) + assert.Equal(t, 10, q.GetWorkerMaxNumber()) + assert.Equal(t, 0, q.GetWorkerNumber()) + assert.Equal(t, 0, q.GetWorkerActiveNumber()) + assert.Equal(t, 0, q.GetQueueItemNumber()) + assert.Equal(t, "int", q.GetItemTypeName()) + + // test inherited config + cfgProvider, err := setting.NewConfigProviderFromData(` +[queue] +TYPE = channel +DATADIR = queues/dir1 +LENGTH = 100 +BATCH_LENGTH = 20 +CONN_STR = "addrs=127.0.0.1:6379 db=0" +QUEUE_NAME = _queue1 + +[queue.sub] +TYPE = level +DATADIR = queues/dir2 +LENGTH = 102 +BATCH_LENGTH = 22 +CONN_STR = +QUEUE_NAME = _q2 +SET_NAME = _u2 +MAX_WORKERS = 2 +`) + + assert.NoError(t, err) + + q1 := createWorkerPoolQueue[string]("no-such", cfgProvider, nil, false) + assert.Equal(t, "no-such", q1.GetName()) + assert.Equal(t, "dummy", q1.GetType()) // no handler, so it becomes dummy + assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/dir1"), q1.baseConfig.DataFullDir) + assert.Equal(t, 100, q1.baseConfig.Length) + assert.Equal(t, 20, q1.batchLength) + assert.Equal(t, "addrs=127.0.0.1:6379 db=0", q1.baseConfig.ConnStr) + assert.Equal(t, "no-such_queue1", q1.baseConfig.QueueFullName) + assert.Equal(t, "no-such_queue1_unique", q1.baseConfig.SetFullName) + assert.Equal(t, 10, q1.GetWorkerMaxNumber()) + assert.Equal(t, 0, q1.GetWorkerNumber()) + assert.Equal(t, 0, q1.GetWorkerActiveNumber()) + assert.Equal(t, 0, q1.GetQueueItemNumber()) + assert.Equal(t, "string", q1.GetItemTypeName()) + qid1 := GetManager().qidCounter + + q2 := createWorkerPoolQueue("sub", cfgProvider, func(s ...int) (unhandled []int) { return nil }, false) + assert.Equal(t, "sub", q2.GetName()) + assert.Equal(t, "level", q2.GetType()) + assert.Equal(t, filepath.Join(setting.AppDataPath, "queues/dir2"), q2.baseConfig.DataFullDir) + assert.Equal(t, 102, q2.baseConfig.Length) + assert.Equal(t, 22, q2.batchLength) + assert.Equal(t, "", q2.baseConfig.ConnStr) + assert.Equal(t, "sub_q2", q2.baseConfig.QueueFullName) + assert.Equal(t, "sub_q2_u2", q2.baseConfig.SetFullName) + assert.Equal(t, 2, q2.GetWorkerMaxNumber()) + assert.Equal(t, 0, q2.GetWorkerNumber()) + assert.Equal(t, 0, q2.GetWorkerActiveNumber()) + assert.Equal(t, 0, q2.GetQueueItemNumber()) + assert.Equal(t, "int", q2.GetItemTypeName()) + qid2 := GetManager().qidCounter + + assert.Equal(t, q1, GetManager().ManagedQueues()[qid1]) + + GetManager().GetManagedQueue(qid1).SetWorkerMaxNumber(120) + assert.Equal(t, 120, q1.workerMaxNum) + + stop := runWorkerPoolQueue(q2) + assert.NoError(t, GetManager().GetManagedQueue(qid2).FlushWithContext(context.Background(), 0)) + assert.NoError(t, GetManager().FlushAll(context.Background(), 0)) + stop() +} diff --git a/modules/queue/queue.go b/modules/queue/queue.go index 22ee64f8e..0ab8dd4ae 100644 --- a/modules/queue/queue.go +++ b/modules/queue/queue.go @@ -1,201 +1,31 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2023 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT +// Package queue implements a specialized queue system for Gitea. +// +// There are two major kinds of concepts: +// +// * The "base queue": channel, level, redis: +// - They have the same abstraction, the same interface, and they are tested by the same testing code. +// - The dummy(immediate) queue is special, it's not a real queue, it's only used as a no-op queue or a testing queue. +// +// * The WorkerPoolQueue: it uses the "base queue" to provide "worker pool" function. +// - It calls the "handler" to process the data in the base queue. +// - Its "Push" function doesn't block forever, +// it will return an error if the queue is full after the timeout. +// +// A queue can be "simple" or "unique". A unique queue will try to avoid duplicate items. +// Unique queue's "Has" function can be used to check whether an item is already in the queue, +// although it's not 100% reliable due to there is no proper transaction support. +// Simple queue's "Has" function always returns "has=false". +// +// The HandlerFuncT function is called by the WorkerPoolQueue to process the data in the base queue. +// If the handler returns "unhandled" items, they will be re-queued to the base queue after a slight delay, +// in case the item processor (eg: document indexer) is not available. package queue -import ( - "context" - "fmt" - "time" -) +import "code.gitea.io/gitea/modules/util" -// ErrInvalidConfiguration is called when there is invalid configuration for a queue -type ErrInvalidConfiguration struct { - cfg interface{} - err error -} +type HandlerFuncT[T any] func(...T) (unhandled []T) -func (err ErrInvalidConfiguration) Error() string { - if err.err != nil { - return fmt.Sprintf("Invalid Configuration Argument: %v: Error: %v", err.cfg, err.err) - } - return fmt.Sprintf("Invalid Configuration Argument: %v", err.cfg) -} - -// IsErrInvalidConfiguration checks if an error is an ErrInvalidConfiguration -func IsErrInvalidConfiguration(err error) bool { - _, ok := err.(ErrInvalidConfiguration) - return ok -} - -// Type is a type of Queue -type Type string - -// Data defines an type of queuable data -type Data interface{} - -// HandlerFunc is a function that takes a variable amount of data and processes it -type HandlerFunc func(...Data) (unhandled []Data) - -// NewQueueFunc is a function that creates a queue -type NewQueueFunc func(handler HandlerFunc, config, exemplar interface{}) (Queue, error) - -// Shutdownable represents a queue that can be shutdown -type Shutdownable interface { - Shutdown() - Terminate() -} - -// Named represents a queue with a name -type Named interface { - Name() string -} - -// Queue defines an interface of a queue-like item -// -// Queues will handle their own contents in the Run method -type Queue interface { - Flushable - Run(atShutdown, atTerminate func(func())) - Push(Data) error -} - -// PushBackable queues can be pushed back to -type PushBackable interface { - // PushBack pushes data back to the top of the fifo - PushBack(Data) error -} - -// DummyQueueType is the type for the dummy queue -const DummyQueueType Type = "dummy" - -// NewDummyQueue creates a new DummyQueue -func NewDummyQueue(handler HandlerFunc, opts, exemplar interface{}) (Queue, error) { - return &DummyQueue{}, nil -} - -// DummyQueue represents an empty queue -type DummyQueue struct{} - -// Run does nothing -func (*DummyQueue) Run(_, _ func(func())) {} - -// Push fakes a push of data to the queue -func (*DummyQueue) Push(Data) error { - return nil -} - -// PushFunc fakes a push of data to the queue with a function. The function is never run. -func (*DummyQueue) PushFunc(Data, func() error) error { - return nil -} - -// Has always returns false as this queue never does anything -func (*DummyQueue) Has(Data) (bool, error) { - return false, nil -} - -// Flush always returns nil -func (*DummyQueue) Flush(time.Duration) error { - return nil -} - -// FlushWithContext always returns nil -func (*DummyQueue) FlushWithContext(context.Context) error { - return nil -} - -// IsEmpty asserts that the queue is empty -func (*DummyQueue) IsEmpty() bool { - return true -} - -// ImmediateType is the type to execute the function when push -const ImmediateType Type = "immediate" - -// NewImmediate creates a new false queue to execute the function when push -func NewImmediate(handler HandlerFunc, opts, exemplar interface{}) (Queue, error) { - return &Immediate{ - handler: handler, - }, nil -} - -// Immediate represents an direct execution queue -type Immediate struct { - handler HandlerFunc -} - -// Run does nothing -func (*Immediate) Run(_, _ func(func())) {} - -// Push fakes a push of data to the queue -func (q *Immediate) Push(data Data) error { - return q.PushFunc(data, nil) -} - -// PushFunc fakes a push of data to the queue with a function. The function is never run. -func (q *Immediate) PushFunc(data Data, f func() error) error { - if f != nil { - if err := f(); err != nil { - return err - } - } - q.handler(data) - return nil -} - -// Has always returns false as this queue never does anything -func (*Immediate) Has(Data) (bool, error) { - return false, nil -} - -// Flush always returns nil -func (*Immediate) Flush(time.Duration) error { - return nil -} - -// FlushWithContext always returns nil -func (*Immediate) FlushWithContext(context.Context) error { - return nil -} - -// IsEmpty asserts that the queue is empty -func (*Immediate) IsEmpty() bool { - return true -} - -var queuesMap = map[Type]NewQueueFunc{ - DummyQueueType: NewDummyQueue, - ImmediateType: NewImmediate, -} - -// RegisteredTypes provides the list of requested types of queues -func RegisteredTypes() []Type { - types := make([]Type, len(queuesMap)) - i := 0 - for key := range queuesMap { - types[i] = key - i++ - } - return types -} - -// RegisteredTypesAsString provides the list of requested types of queues -func RegisteredTypesAsString() []string { - types := make([]string, len(queuesMap)) - i := 0 - for key := range queuesMap { - types[i] = string(key) - i++ - } - return types -} - -// NewQueue takes a queue Type, HandlerFunc, some options and possibly an exemplar and returns a Queue or an error -func NewQueue(queueType Type, handlerFunc HandlerFunc, opts, exemplar interface{}) (Queue, error) { - newFn, ok := queuesMap[queueType] - if !ok { - return nil, fmt.Errorf("unsupported queue type: %v", queueType) - } - return newFn(handlerFunc, opts, exemplar) -} +var ErrAlreadyInQueue = util.NewAlreadyExistErrorf("already in queue") diff --git a/modules/queue/queue_bytefifo.go b/modules/queue/queue_bytefifo.go deleted file mode 100644 index ee00a5428..000000000 --- a/modules/queue/queue_bytefifo.go +++ /dev/null @@ -1,419 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "fmt" - "runtime/pprof" - "sync" - "sync/atomic" - "time" - - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" -) - -// ByteFIFOQueueConfiguration is the configuration for a ByteFIFOQueue -type ByteFIFOQueueConfiguration struct { - WorkerPoolConfiguration - Workers int - WaitOnEmpty bool -} - -var _ Queue = &ByteFIFOQueue{} - -// ByteFIFOQueue is a Queue formed from a ByteFIFO and WorkerPool -type ByteFIFOQueue struct { - *WorkerPool - byteFIFO ByteFIFO - typ Type - shutdownCtx context.Context - shutdownCtxCancel context.CancelFunc - terminateCtx context.Context - terminateCtxCancel context.CancelFunc - exemplar interface{} - workers int - name string - lock sync.Mutex - waitOnEmpty bool - pushed chan struct{} -} - -// NewByteFIFOQueue creates a new ByteFIFOQueue -func NewByteFIFOQueue(typ Type, byteFIFO ByteFIFO, handle HandlerFunc, cfg, exemplar interface{}) (*ByteFIFOQueue, error) { - configInterface, err := toConfig(ByteFIFOQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(ByteFIFOQueueConfiguration) - - terminateCtx, terminateCtxCancel := context.WithCancel(context.Background()) - shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx) - - q := &ByteFIFOQueue{ - byteFIFO: byteFIFO, - typ: typ, - shutdownCtx: shutdownCtx, - shutdownCtxCancel: shutdownCtxCancel, - terminateCtx: terminateCtx, - terminateCtxCancel: terminateCtxCancel, - exemplar: exemplar, - workers: config.Workers, - name: config.Name, - waitOnEmpty: config.WaitOnEmpty, - pushed: make(chan struct{}, 1), - } - q.WorkerPool = NewWorkerPool(func(data ...Data) (failed []Data) { - for _, unhandled := range handle(data...) { - if fail := q.PushBack(unhandled); fail != nil { - failed = append(failed, fail) - } - } - return failed - }, config.WorkerPoolConfiguration) - - return q, nil -} - -// Name returns the name of this queue -func (q *ByteFIFOQueue) Name() string { - return q.name -} - -// Push pushes data to the fifo -func (q *ByteFIFOQueue) Push(data Data) error { - return q.PushFunc(data, nil) -} - -// PushBack pushes data to the fifo -func (q *ByteFIFOQueue) PushBack(data Data) error { - if !assignableTo(data, q.exemplar) { - return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) - } - bs, err := json.Marshal(data) - if err != nil { - return err - } - defer func() { - select { - case q.pushed <- struct{}{}: - default: - } - }() - return q.byteFIFO.PushBack(q.terminateCtx, bs) -} - -// PushFunc pushes data to the fifo -func (q *ByteFIFOQueue) PushFunc(data Data, fn func() error) error { - if !assignableTo(data, q.exemplar) { - return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) - } - bs, err := json.Marshal(data) - if err != nil { - return err - } - defer func() { - select { - case q.pushed <- struct{}{}: - default: - } - }() - return q.byteFIFO.PushFunc(q.terminateCtx, bs, fn) -} - -// IsEmpty checks if the queue is empty -func (q *ByteFIFOQueue) IsEmpty() bool { - q.lock.Lock() - defer q.lock.Unlock() - if !q.WorkerPool.IsEmpty() { - return false - } - return q.byteFIFO.Len(q.terminateCtx) == 0 -} - -// NumberInQueue returns the number in the queue -func (q *ByteFIFOQueue) NumberInQueue() int64 { - q.lock.Lock() - defer q.lock.Unlock() - return q.byteFIFO.Len(q.terminateCtx) + q.WorkerPool.NumberInQueue() -} - -// Flush flushes the ByteFIFOQueue -func (q *ByteFIFOQueue) Flush(timeout time.Duration) error { - select { - case q.pushed <- struct{}{}: - default: - } - return q.WorkerPool.Flush(timeout) -} - -// Run runs the bytefifo queue -func (q *ByteFIFOQueue) Run(atShutdown, atTerminate func(func())) { - pprof.SetGoroutineLabels(q.baseCtx) - atShutdown(q.Shutdown) - atTerminate(q.Terminate) - log.Debug("%s: %s Starting", q.typ, q.name) - - _ = q.AddWorkers(q.workers, 0) - - log.Trace("%s: %s Now running", q.typ, q.name) - q.readToChan() - - <-q.shutdownCtx.Done() - log.Trace("%s: %s Waiting til done", q.typ, q.name) - q.Wait() - - log.Trace("%s: %s Waiting til cleaned", q.typ, q.name) - q.CleanUp(q.terminateCtx) - q.terminateCtxCancel() -} - -const maxBackOffTime = time.Second * 3 - -func (q *ByteFIFOQueue) readToChan() { - // handle quick cancels - select { - case <-q.shutdownCtx.Done(): - // tell the pool to shutdown. - q.baseCtxCancel() - return - default: - } - - // Default backoff values - backOffTime := time.Millisecond * 100 - backOffTimer := time.NewTimer(0) - util.StopTimer(backOffTimer) - - paused, _ := q.IsPausedIsResumed() - -loop: - for { - select { - case <-paused: - log.Trace("Queue %s pausing", q.name) - _, resumed := q.IsPausedIsResumed() - - select { - case <-resumed: - paused, _ = q.IsPausedIsResumed() - log.Trace("Queue %s resuming", q.name) - if q.HasNoWorkerScaling() { - log.Warn( - "Queue: %s is configured to be non-scaling and has no workers - this configuration is likely incorrect.\n"+ - "The queue will be paused to prevent data-loss with the assumption that you will add workers and unpause as required.", q.name) - q.Pause() - continue loop - } - case <-q.shutdownCtx.Done(): - // tell the pool to shutdown. - q.baseCtxCancel() - return - case data, ok := <-q.dataChan: - if !ok { - return - } - if err := q.PushBack(data); err != nil { - log.Error("Unable to push back data into queue %s", q.name) - } - atomic.AddInt64(&q.numInQueue, -1) - } - default: - } - - // empty the pushed channel - select { - case <-q.pushed: - default: - } - - err := q.doPop() - - util.StopTimer(backOffTimer) - - if err != nil { - if err == errQueueEmpty && q.waitOnEmpty { - log.Trace("%s: %s Waiting on Empty", q.typ, q.name) - - // reset the backoff time but don't set the timer - backOffTime = 100 * time.Millisecond - } else if err == errUnmarshal { - // reset the timer and backoff - backOffTime = 100 * time.Millisecond - backOffTimer.Reset(backOffTime) - } else { - // backoff - backOffTimer.Reset(backOffTime) - } - - // Need to Backoff - select { - case <-q.shutdownCtx.Done(): - // Oops we've been shutdown whilst backing off - // Make sure the worker pool is shutdown too - q.baseCtxCancel() - return - case <-q.pushed: - // Data has been pushed to the fifo (or flush has been called) - // reset the backoff time - backOffTime = 100 * time.Millisecond - continue loop - case <-backOffTimer.C: - // Calculate the next backoff time - backOffTime += backOffTime / 2 - if backOffTime > maxBackOffTime { - backOffTime = maxBackOffTime - } - continue loop - } - } - - // Reset the backoff time - backOffTime = 100 * time.Millisecond - - select { - case <-q.shutdownCtx.Done(): - // Oops we've been shutdown - // Make sure the worker pool is shutdown too - q.baseCtxCancel() - return - default: - continue loop - } - } -} - -var ( - errQueueEmpty = fmt.Errorf("empty queue") - errEmptyBytes = fmt.Errorf("empty bytes") - errUnmarshal = fmt.Errorf("failed to unmarshal") -) - -func (q *ByteFIFOQueue) doPop() error { - q.lock.Lock() - defer q.lock.Unlock() - bs, err := q.byteFIFO.Pop(q.shutdownCtx) - if err != nil { - if err == context.Canceled { - q.baseCtxCancel() - return err - } - log.Error("%s: %s Error on Pop: %v", q.typ, q.name, err) - return err - } - if len(bs) == 0 { - if q.waitOnEmpty && q.byteFIFO.Len(q.shutdownCtx) == 0 { - return errQueueEmpty - } - return errEmptyBytes - } - - data, err := unmarshalAs(bs, q.exemplar) - if err != nil { - log.Error("%s: %s Failed to unmarshal with error: %v", q.typ, q.name, err) - return errUnmarshal - } - - log.Trace("%s %s: Task found: %#v", q.typ, q.name, data) - q.WorkerPool.Push(data) - return nil -} - -// Shutdown processing from this queue -func (q *ByteFIFOQueue) Shutdown() { - log.Trace("%s: %s Shutting down", q.typ, q.name) - select { - case <-q.shutdownCtx.Done(): - return - default: - } - q.shutdownCtxCancel() - log.Debug("%s: %s Shutdown", q.typ, q.name) -} - -// IsShutdown returns a channel which is closed when this Queue is shutdown -func (q *ByteFIFOQueue) IsShutdown() <-chan struct{} { - return q.shutdownCtx.Done() -} - -// Terminate this queue and close the queue -func (q *ByteFIFOQueue) Terminate() { - log.Trace("%s: %s Terminating", q.typ, q.name) - q.Shutdown() - select { - case <-q.terminateCtx.Done(): - return - default: - } - if log.IsDebug() { - log.Debug("%s: %s Closing with %d tasks left in queue", q.typ, q.name, q.byteFIFO.Len(q.terminateCtx)) - } - q.terminateCtxCancel() - if err := q.byteFIFO.Close(); err != nil { - log.Error("Error whilst closing internal byte fifo in %s: %s: %v", q.typ, q.name, err) - } - q.baseCtxFinished() - log.Debug("%s: %s Terminated", q.typ, q.name) -} - -// IsTerminated returns a channel which is closed when this Queue is terminated -func (q *ByteFIFOQueue) IsTerminated() <-chan struct{} { - return q.terminateCtx.Done() -} - -var _ UniqueQueue = &ByteFIFOUniqueQueue{} - -// ByteFIFOUniqueQueue represents a UniqueQueue formed from a UniqueByteFifo -type ByteFIFOUniqueQueue struct { - ByteFIFOQueue -} - -// NewByteFIFOUniqueQueue creates a new ByteFIFOUniqueQueue -func NewByteFIFOUniqueQueue(typ Type, byteFIFO UniqueByteFIFO, handle HandlerFunc, cfg, exemplar interface{}) (*ByteFIFOUniqueQueue, error) { - configInterface, err := toConfig(ByteFIFOQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(ByteFIFOQueueConfiguration) - terminateCtx, terminateCtxCancel := context.WithCancel(context.Background()) - shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx) - - q := &ByteFIFOUniqueQueue{ - ByteFIFOQueue: ByteFIFOQueue{ - byteFIFO: byteFIFO, - typ: typ, - shutdownCtx: shutdownCtx, - shutdownCtxCancel: shutdownCtxCancel, - terminateCtx: terminateCtx, - terminateCtxCancel: terminateCtxCancel, - exemplar: exemplar, - workers: config.Workers, - name: config.Name, - }, - } - q.WorkerPool = NewWorkerPool(func(data ...Data) (failed []Data) { - for _, unhandled := range handle(data...) { - if fail := q.PushBack(unhandled); fail != nil { - failed = append(failed, fail) - } - } - return failed - }, config.WorkerPoolConfiguration) - - return q, nil -} - -// Has checks if the provided data is in the queue -func (q *ByteFIFOUniqueQueue) Has(data Data) (bool, error) { - if !assignableTo(data, q.exemplar) { - return false, fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) - } - bs, err := json.Marshal(data) - if err != nil { - return false, err - } - return q.byteFIFO.(UniqueByteFIFO).Has(q.terminateCtx, bs) -} diff --git a/modules/queue/queue_channel.go b/modules/queue/queue_channel.go deleted file mode 100644 index baac09739..000000000 --- a/modules/queue/queue_channel.go +++ /dev/null @@ -1,160 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "fmt" - "runtime/pprof" - "sync/atomic" - "time" - - "code.gitea.io/gitea/modules/log" -) - -// ChannelQueueType is the type for channel queue -const ChannelQueueType Type = "channel" - -// ChannelQueueConfiguration is the configuration for a ChannelQueue -type ChannelQueueConfiguration struct { - WorkerPoolConfiguration - Workers int -} - -// ChannelQueue implements Queue -// -// A channel queue is not persistable and does not shutdown or terminate cleanly -// It is basically a very thin wrapper around a WorkerPool -type ChannelQueue struct { - *WorkerPool - shutdownCtx context.Context - shutdownCtxCancel context.CancelFunc - terminateCtx context.Context - terminateCtxCancel context.CancelFunc - exemplar interface{} - workers int - name string -} - -// NewChannelQueue creates a memory channel queue -func NewChannelQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(ChannelQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(ChannelQueueConfiguration) - if config.BatchLength == 0 { - config.BatchLength = 1 - } - - terminateCtx, terminateCtxCancel := context.WithCancel(context.Background()) - shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx) - - queue := &ChannelQueue{ - shutdownCtx: shutdownCtx, - shutdownCtxCancel: shutdownCtxCancel, - terminateCtx: terminateCtx, - terminateCtxCancel: terminateCtxCancel, - exemplar: exemplar, - workers: config.Workers, - name: config.Name, - } - queue.WorkerPool = NewWorkerPool(func(data ...Data) []Data { - unhandled := handle(data...) - if len(unhandled) > 0 { - // We can only pushback to the channel if we're paused. - if queue.IsPaused() { - atomic.AddInt64(&queue.numInQueue, int64(len(unhandled))) - go func() { - for _, datum := range data { - queue.dataChan <- datum - } - }() - return nil - } - } - return unhandled - }, config.WorkerPoolConfiguration) - - queue.qid = GetManager().Add(queue, ChannelQueueType, config, exemplar) - return queue, nil -} - -// Run starts to run the queue -func (q *ChannelQueue) Run(atShutdown, atTerminate func(func())) { - pprof.SetGoroutineLabels(q.baseCtx) - atShutdown(q.Shutdown) - atTerminate(q.Terminate) - log.Debug("ChannelQueue: %s Starting", q.name) - _ = q.AddWorkers(q.workers, 0) -} - -// Push will push data into the queue -func (q *ChannelQueue) Push(data Data) error { - if !assignableTo(data, q.exemplar) { - return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in queue: %s", data, q.exemplar, q.name) - } - q.WorkerPool.Push(data) - return nil -} - -// Flush flushes the channel with a timeout - the Flush worker will be registered as a flush worker with the manager -func (q *ChannelQueue) Flush(timeout time.Duration) error { - if q.IsPaused() { - return nil - } - ctx, cancel := q.commonRegisterWorkers(1, timeout, true) - defer cancel() - return q.FlushWithContext(ctx) -} - -// Shutdown processing from this queue -func (q *ChannelQueue) Shutdown() { - q.lock.Lock() - defer q.lock.Unlock() - select { - case <-q.shutdownCtx.Done(): - log.Trace("ChannelQueue: %s Already Shutting down", q.name) - return - default: - } - log.Trace("ChannelQueue: %s Shutting down", q.name) - go func() { - log.Trace("ChannelQueue: %s Flushing", q.name) - // We can't use Cleanup here because that will close the channel - if err := q.FlushWithContext(q.terminateCtx); err != nil { - count := atomic.LoadInt64(&q.numInQueue) - if count > 0 { - log.Warn("ChannelQueue: %s Terminated before completed flushing", q.name) - } - return - } - log.Debug("ChannelQueue: %s Flushed", q.name) - }() - q.shutdownCtxCancel() - log.Debug("ChannelQueue: %s Shutdown", q.name) -} - -// Terminate this queue and close the queue -func (q *ChannelQueue) Terminate() { - log.Trace("ChannelQueue: %s Terminating", q.name) - q.Shutdown() - select { - case <-q.terminateCtx.Done(): - return - default: - } - q.terminateCtxCancel() - q.baseCtxFinished() - log.Debug("ChannelQueue: %s Terminated", q.name) -} - -// Name returns the name of this queue -func (q *ChannelQueue) Name() string { - return q.name -} - -func init() { - queuesMap[ChannelQueueType] = NewChannelQueue -} diff --git a/modules/queue/queue_channel_test.go b/modules/queue/queue_channel_test.go deleted file mode 100644 index f9dae742e..000000000 --- a/modules/queue/queue_channel_test.go +++ /dev/null @@ -1,315 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "os" - "sync" - "testing" - "time" - - "code.gitea.io/gitea/modules/log" - - "github.com/stretchr/testify/assert" -) - -func TestChannelQueue(t *testing.T) { - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - nilFn := func(_ func()) {} - - queue, err := NewChannelQueue(handle, - ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 0, - MaxWorkers: 10, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 5, - Name: "TestChannelQueue", - }, - Workers: 0, - }, &testData{}) - assert.NoError(t, err) - - assert.Equal(t, 5, queue.(*ChannelQueue).WorkerPool.boostWorkers) - - go queue.Run(nilFn, nilFn) - - test1 := testData{"A", 1} - go queue.Push(&test1) - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - err = queue.Push(test1) - assert.Error(t, err) -} - -func TestChannelQueue_Batch(t *testing.T) { - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - assert.True(t, len(data) == 2) - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - nilFn := func(_ func()) {} - - queue, err := NewChannelQueue(handle, - ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 20, - BatchLength: 2, - BlockTimeout: 0, - BoostTimeout: 0, - BoostWorkers: 0, - MaxWorkers: 10, - }, - Workers: 1, - }, &testData{}) - assert.NoError(t, err) - - go queue.Run(nilFn, nilFn) - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - - queue.Push(&test1) - go queue.Push(&test2) - - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - result2 := <-handleChan - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - err = queue.Push(test1) - assert.Error(t, err) -} - -func TestChannelQueue_Pause(t *testing.T) { - if os.Getenv("CI") != "" { - t.Skip("Skipping because test is flaky on CI") - } - lock := sync.Mutex{} - var queue Queue - var err error - pushBack := false - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - lock.Lock() - if pushBack { - if pausable, ok := queue.(Pausable); ok { - pausable.Pause() - } - lock.Unlock() - return data - } - lock.Unlock() - - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - queueShutdown := []func(){} - queueTerminate := []func(){} - - terminated := make(chan struct{}) - - queue, err = NewChannelQueue(handle, - ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 20, - BatchLength: 1, - BlockTimeout: 0, - BoostTimeout: 0, - BoostWorkers: 0, - MaxWorkers: 10, - }, - Workers: 1, - }, &testData{}) - assert.NoError(t, err) - - go func() { - queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - queueTerminate = append(queueTerminate, terminate) - }) - close(terminated) - }() - - // Shutdown and Terminate in defer - defer func() { - lock.Lock() - callbacks := make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - lock.Lock() - log.Info("Finally terminating") - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - }() - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - queue.Push(&test1) - - pausable, ok := queue.(Pausable) - if !assert.True(t, ok) { - return - } - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - pausable.Pause() - - paused, _ := pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue is not paused") - return - } - - queue.Push(&test2) - - var result2 *testData - select { - case result2 = <-handleChan: - assert.Fail(t, "handler chan should be empty") - case <-time.After(100 * time.Millisecond): - } - - assert.Nil(t, result2) - - pausable.Resume() - _, resumed := pausable.IsPausedIsResumed() - - select { - case <-resumed: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue should be resumed") - } - - select { - case result2 = <-handleChan: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "handler chan should contain test2") - } - - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - lock.Lock() - pushBack = true - lock.Unlock() - - _, resumed = pausable.IsPausedIsResumed() - - select { - case <-resumed: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue is not resumed") - return - } - - queue.Push(&test1) - paused, _ = pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-handleChan: - assert.Fail(t, "handler chan should not contain test1") - return - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "queue should be paused") - return - } - - lock.Lock() - pushBack = false - lock.Unlock() - - paused, _ = pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue is not paused") - return - } - - pausable.Resume() - _, resumed = pausable.IsPausedIsResumed() - - select { - case <-resumed: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue should be resumed") - } - - select { - case result1 = <-handleChan: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "handler chan should contain test1") - } - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - lock.Lock() - callbacks := make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - queueShutdown = queueShutdown[:0] - lock.Unlock() - // Now shutdown the queue - for _, callback := range callbacks { - callback() - } - - // terminate the queue - lock.Lock() - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - queueShutdown = queueTerminate[:0] - lock.Unlock() - for _, callback := range callbacks { - callback() - } - select { - case <-terminated: - case <-time.After(10 * time.Second): - assert.Fail(t, "Queue should have terminated") - return - } -} diff --git a/modules/queue/queue_disk.go b/modules/queue/queue_disk.go deleted file mode 100644 index fbedb8e5b..000000000 --- a/modules/queue/queue_disk.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - - "code.gitea.io/gitea/modules/nosql" - - "gitea.com/lunny/levelqueue" -) - -// LevelQueueType is the type for level queue -const LevelQueueType Type = "level" - -// LevelQueueConfiguration is the configuration for a LevelQueue -type LevelQueueConfiguration struct { - ByteFIFOQueueConfiguration - DataDir string - ConnectionString string - QueueName string -} - -// LevelQueue implements a disk library queue -type LevelQueue struct { - *ByteFIFOQueue -} - -// NewLevelQueue creates a ledis local queue -func NewLevelQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(LevelQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(LevelQueueConfiguration) - - if len(config.ConnectionString) == 0 { - config.ConnectionString = config.DataDir - } - config.WaitOnEmpty = true - - byteFIFO, err := NewLevelQueueByteFIFO(config.ConnectionString, config.QueueName) - if err != nil { - return nil, err - } - - byteFIFOQueue, err := NewByteFIFOQueue(LevelQueueType, byteFIFO, handle, config.ByteFIFOQueueConfiguration, exemplar) - if err != nil { - return nil, err - } - - queue := &LevelQueue{ - ByteFIFOQueue: byteFIFOQueue, - } - queue.qid = GetManager().Add(queue, LevelQueueType, config, exemplar) - return queue, nil -} - -var _ ByteFIFO = &LevelQueueByteFIFO{} - -// LevelQueueByteFIFO represents a ByteFIFO formed from a LevelQueue -type LevelQueueByteFIFO struct { - internal *levelqueue.Queue - connection string -} - -// NewLevelQueueByteFIFO creates a ByteFIFO formed from a LevelQueue -func NewLevelQueueByteFIFO(connection, prefix string) (*LevelQueueByteFIFO, error) { - db, err := nosql.GetManager().GetLevelDB(connection) - if err != nil { - return nil, err - } - - internal, err := levelqueue.NewQueue(db, []byte(prefix), false) - if err != nil { - return nil, err - } - - return &LevelQueueByteFIFO{ - connection: connection, - internal: internal, - }, nil -} - -// PushFunc will push data into the fifo -func (fifo *LevelQueueByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { - if fn != nil { - if err := fn(); err != nil { - return err - } - } - return fifo.internal.LPush(data) -} - -// PushBack pushes data to the top of the fifo -func (fifo *LevelQueueByteFIFO) PushBack(ctx context.Context, data []byte) error { - return fifo.internal.RPush(data) -} - -// Pop pops data from the start of the fifo -func (fifo *LevelQueueByteFIFO) Pop(ctx context.Context) ([]byte, error) { - data, err := fifo.internal.RPop() - if err != nil && err != levelqueue.ErrNotFound { - return nil, err - } - return data, nil -} - -// Close this fifo -func (fifo *LevelQueueByteFIFO) Close() error { - err := fifo.internal.Close() - _ = nosql.GetManager().CloseLevelDB(fifo.connection) - return err -} - -// Len returns the length of the fifo -func (fifo *LevelQueueByteFIFO) Len(ctx context.Context) int64 { - return fifo.internal.Len() -} - -func init() { - queuesMap[LevelQueueType] = NewLevelQueue -} diff --git a/modules/queue/queue_disk_channel.go b/modules/queue/queue_disk_channel.go deleted file mode 100644 index 91f91f0df..000000000 --- a/modules/queue/queue_disk_channel.go +++ /dev/null @@ -1,358 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "fmt" - "runtime/pprof" - "sync" - "sync/atomic" - "time" - - "code.gitea.io/gitea/modules/log" -) - -// PersistableChannelQueueType is the type for persistable queue -const PersistableChannelQueueType Type = "persistable-channel" - -// PersistableChannelQueueConfiguration is the configuration for a PersistableChannelQueue -type PersistableChannelQueueConfiguration struct { - Name string - DataDir string - BatchLength int - QueueLength int - Timeout time.Duration - MaxAttempts int - Workers int - MaxWorkers int - BlockTimeout time.Duration - BoostTimeout time.Duration - BoostWorkers int -} - -// PersistableChannelQueue wraps a channel queue and level queue together -// The disk level queue will be used to store data at shutdown and terminate - and will be restored -// on start up. -type PersistableChannelQueue struct { - channelQueue *ChannelQueue - delayedStarter - lock sync.Mutex - closed chan struct{} -} - -// NewPersistableChannelQueue creates a wrapped batched channel queue with persistable level queue backend when shutting down -// This differs from a wrapped queue in that the persistent queue is only used to persist at shutdown/terminate -func NewPersistableChannelQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(PersistableChannelQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(PersistableChannelQueueConfiguration) - - queue := &PersistableChannelQueue{ - closed: make(chan struct{}), - } - - wrappedHandle := func(data ...Data) (failed []Data) { - for _, unhandled := range handle(data...) { - if fail := queue.PushBack(unhandled); fail != nil { - failed = append(failed, fail) - } - } - return failed - } - - channelQueue, err := NewChannelQueue(wrappedHandle, ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: config.QueueLength, - BatchLength: config.BatchLength, - BlockTimeout: config.BlockTimeout, - BoostTimeout: config.BoostTimeout, - BoostWorkers: config.BoostWorkers, - MaxWorkers: config.MaxWorkers, - Name: config.Name + "-channel", - }, - Workers: config.Workers, - }, exemplar) - if err != nil { - return nil, err - } - - // the level backend only needs temporary workers to catch up with the previously dropped work - levelCfg := LevelQueueConfiguration{ - ByteFIFOQueueConfiguration: ByteFIFOQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: config.QueueLength, - BatchLength: config.BatchLength, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 1, - MaxWorkers: 5, - Name: config.Name + "-level", - }, - Workers: 0, - }, - DataDir: config.DataDir, - QueueName: config.Name + "-level", - } - - levelQueue, err := NewLevelQueue(wrappedHandle, levelCfg, exemplar) - if err == nil { - queue.channelQueue = channelQueue.(*ChannelQueue) - queue.delayedStarter = delayedStarter{ - internal: levelQueue.(*LevelQueue), - name: config.Name, - } - _ = GetManager().Add(queue, PersistableChannelQueueType, config, exemplar) - return queue, nil - } - if IsErrInvalidConfiguration(err) { - // Retrying ain't gonna make this any better... - return nil, ErrInvalidConfiguration{cfg: cfg} - } - - queue.channelQueue = channelQueue.(*ChannelQueue) - queue.delayedStarter = delayedStarter{ - cfg: levelCfg, - underlying: LevelQueueType, - timeout: config.Timeout, - maxAttempts: config.MaxAttempts, - name: config.Name, - } - _ = GetManager().Add(queue, PersistableChannelQueueType, config, exemplar) - return queue, nil -} - -// Name returns the name of this queue -func (q *PersistableChannelQueue) Name() string { - return q.delayedStarter.name -} - -// Push will push the indexer data to queue -func (q *PersistableChannelQueue) Push(data Data) error { - select { - case <-q.closed: - return q.internal.Push(data) - default: - return q.channelQueue.Push(data) - } -} - -// PushBack will push the indexer data to queue -func (q *PersistableChannelQueue) PushBack(data Data) error { - select { - case <-q.closed: - if pbr, ok := q.internal.(PushBackable); ok { - return pbr.PushBack(data) - } - return q.internal.Push(data) - default: - return q.channelQueue.Push(data) - } -} - -// Run starts to run the queue -func (q *PersistableChannelQueue) Run(atShutdown, atTerminate func(func())) { - pprof.SetGoroutineLabels(q.channelQueue.baseCtx) - log.Debug("PersistableChannelQueue: %s Starting", q.delayedStarter.name) - _ = q.channelQueue.AddWorkers(q.channelQueue.workers, 0) - - q.lock.Lock() - if q.internal == nil { - err := q.setInternal(atShutdown, q.channelQueue.handle, q.channelQueue.exemplar) - q.lock.Unlock() - if err != nil { - log.Fatal("Unable to create internal queue for %s Error: %v", q.Name(), err) - return - } - } else { - q.lock.Unlock() - } - atShutdown(q.Shutdown) - atTerminate(q.Terminate) - - if lq, ok := q.internal.(*LevelQueue); ok && lq.byteFIFO.Len(lq.terminateCtx) != 0 { - // Just run the level queue - we shut it down once it's flushed - go q.internal.Run(func(_ func()) {}, func(_ func()) {}) - go func() { - for !lq.IsEmpty() { - _ = lq.Flush(0) - select { - case <-time.After(100 * time.Millisecond): - case <-lq.shutdownCtx.Done(): - if lq.byteFIFO.Len(lq.terminateCtx) > 0 { - log.Warn("LevelQueue: %s shut down before completely flushed", q.internal.(*LevelQueue).Name()) - } - return - } - } - log.Debug("LevelQueue: %s flushed so shutting down", q.internal.(*LevelQueue).Name()) - q.internal.(*LevelQueue).Shutdown() - GetManager().Remove(q.internal.(*LevelQueue).qid) - }() - } else { - log.Debug("PersistableChannelQueue: %s Skipping running the empty level queue", q.delayedStarter.name) - q.internal.(*LevelQueue).Shutdown() - GetManager().Remove(q.internal.(*LevelQueue).qid) - } -} - -// Flush flushes the queue and blocks till the queue is empty -func (q *PersistableChannelQueue) Flush(timeout time.Duration) error { - var ctx context.Context - var cancel context.CancelFunc - if timeout > 0 { - ctx, cancel = context.WithTimeout(context.Background(), timeout) - } else { - ctx, cancel = context.WithCancel(context.Background()) - } - defer cancel() - return q.FlushWithContext(ctx) -} - -// FlushWithContext flushes the queue and blocks till the queue is empty -func (q *PersistableChannelQueue) FlushWithContext(ctx context.Context) error { - errChan := make(chan error, 1) - go func() { - errChan <- q.channelQueue.FlushWithContext(ctx) - }() - go func() { - q.lock.Lock() - if q.internal == nil { - q.lock.Unlock() - errChan <- fmt.Errorf("not ready to flush internal queue %s yet", q.Name()) - return - } - q.lock.Unlock() - errChan <- q.internal.FlushWithContext(ctx) - }() - err1 := <-errChan - err2 := <-errChan - - if err1 != nil { - return err1 - } - return err2 -} - -// IsEmpty checks if a queue is empty -func (q *PersistableChannelQueue) IsEmpty() bool { - if !q.channelQueue.IsEmpty() { - return false - } - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return false - } - return q.internal.IsEmpty() -} - -// IsPaused returns if the pool is paused -func (q *PersistableChannelQueue) IsPaused() bool { - return q.channelQueue.IsPaused() -} - -// IsPausedIsResumed returns if the pool is paused and a channel that is closed when it is resumed -func (q *PersistableChannelQueue) IsPausedIsResumed() (<-chan struct{}, <-chan struct{}) { - return q.channelQueue.IsPausedIsResumed() -} - -// Pause pauses the WorkerPool -func (q *PersistableChannelQueue) Pause() { - q.channelQueue.Pause() - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return - } - - pausable, ok := q.internal.(Pausable) - if !ok { - return - } - pausable.Pause() -} - -// Resume resumes the WorkerPool -func (q *PersistableChannelQueue) Resume() { - q.channelQueue.Resume() - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return - } - - pausable, ok := q.internal.(Pausable) - if !ok { - return - } - pausable.Resume() -} - -// Shutdown processing this queue -func (q *PersistableChannelQueue) Shutdown() { - log.Trace("PersistableChannelQueue: %s Shutting down", q.delayedStarter.name) - q.lock.Lock() - - select { - case <-q.closed: - q.lock.Unlock() - return - default: - } - q.channelQueue.Shutdown() - if q.internal != nil { - q.internal.(*LevelQueue).Shutdown() - } - close(q.closed) - q.lock.Unlock() - - log.Trace("PersistableChannelQueue: %s Cancelling pools", q.delayedStarter.name) - q.channelQueue.baseCtxCancel() - q.internal.(*LevelQueue).baseCtxCancel() - log.Trace("PersistableChannelQueue: %s Waiting til done", q.delayedStarter.name) - q.channelQueue.Wait() - q.internal.(*LevelQueue).Wait() - // Redirect all remaining data in the chan to the internal channel - log.Trace("PersistableChannelQueue: %s Redirecting remaining data", q.delayedStarter.name) - close(q.channelQueue.dataChan) - countOK, countLost := 0, 0 - for data := range q.channelQueue.dataChan { - err := q.internal.Push(data) - if err != nil { - log.Error("PersistableChannelQueue: %s Unable redirect %v due to: %v", q.delayedStarter.name, data, err) - countLost++ - } else { - countOK++ - } - atomic.AddInt64(&q.channelQueue.numInQueue, -1) - } - if countLost > 0 { - log.Warn("PersistableChannelQueue: %s %d will be restored on restart, %d lost", q.delayedStarter.name, countOK, countLost) - } else if countOK > 0 { - log.Warn("PersistableChannelQueue: %s %d will be restored on restart", q.delayedStarter.name, countOK) - } - log.Trace("PersistableChannelQueue: %s Done Redirecting remaining data", q.delayedStarter.name) - - log.Debug("PersistableChannelQueue: %s Shutdown", q.delayedStarter.name) -} - -// Terminate this queue and close the queue -func (q *PersistableChannelQueue) Terminate() { - log.Trace("PersistableChannelQueue: %s Terminating", q.delayedStarter.name) - q.Shutdown() - q.lock.Lock() - defer q.lock.Unlock() - q.channelQueue.Terminate() - if q.internal != nil { - q.internal.(*LevelQueue).Terminate() - } - log.Debug("PersistableChannelQueue: %s Terminated", q.delayedStarter.name) -} - -func init() { - queuesMap[PersistableChannelQueueType] = NewPersistableChannelQueue -} diff --git a/modules/queue/queue_disk_channel_test.go b/modules/queue/queue_disk_channel_test.go deleted file mode 100644 index 4f14a5d79..000000000 --- a/modules/queue/queue_disk_channel_test.go +++ /dev/null @@ -1,544 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "sync" - "testing" - "time" - - "code.gitea.io/gitea/modules/log" - - "github.com/stretchr/testify/assert" -) - -func TestPersistableChannelQueue(t *testing.T) { - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - for _, datum := range data { - if datum == nil { - continue - } - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - lock := sync.Mutex{} - queueShutdown := []func(){} - queueTerminate := []func(){} - - tmpDir := t.TempDir() - - queue, err := NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ - DataDir: tmpDir, - BatchLength: 2, - QueueLength: 20, - Workers: 1, - BoostWorkers: 0, - MaxWorkers: 10, - Name: "test-queue", - }, &testData{}) - assert.NoError(t, err) - - readyForShutdown := make(chan struct{}) - readyForTerminate := make(chan struct{}) - - go queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - select { - case <-readyForShutdown: - default: - close(readyForShutdown) - } - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - select { - case <-readyForTerminate: - default: - close(readyForTerminate) - } - queueTerminate = append(queueTerminate, terminate) - }) - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - - err = queue.Push(&test1) - assert.NoError(t, err) - go func() { - err := queue.Push(&test2) - assert.NoError(t, err) - }() - - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - result2 := <-handleChan - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - // test1 is a testData not a *testData so will be rejected - err = queue.Push(test1) - assert.Error(t, err) - - <-readyForShutdown - // Now shutdown the queue - lock.Lock() - callbacks := make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - - // Wait til it is closed - <-queue.(*PersistableChannelQueue).closed - - err = queue.Push(&test1) - assert.NoError(t, err) - err = queue.Push(&test2) - assert.NoError(t, err) - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - default: - } - - // terminate the queue - <-readyForTerminate - lock.Lock() - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - default: - } - - // Reopen queue - queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ - DataDir: tmpDir, - BatchLength: 2, - QueueLength: 20, - Workers: 1, - BoostWorkers: 0, - MaxWorkers: 10, - Name: "test-queue", - }, &testData{}) - assert.NoError(t, err) - - readyForShutdown = make(chan struct{}) - readyForTerminate = make(chan struct{}) - - go queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - select { - case <-readyForShutdown: - default: - close(readyForShutdown) - } - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - select { - case <-readyForTerminate: - default: - close(readyForTerminate) - } - queueTerminate = append(queueTerminate, terminate) - }) - - result3 := <-handleChan - assert.Equal(t, test1.TestString, result3.TestString) - assert.Equal(t, test1.TestInt, result3.TestInt) - - result4 := <-handleChan - assert.Equal(t, test2.TestString, result4.TestString) - assert.Equal(t, test2.TestInt, result4.TestInt) - - <-readyForShutdown - lock.Lock() - callbacks = make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - <-readyForTerminate - lock.Lock() - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - lock.Unlock() - for _, callback := range callbacks { - callback() - } -} - -func TestPersistableChannelQueue_Pause(t *testing.T) { - lock := sync.Mutex{} - var queue Queue - var err error - pushBack := false - - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - lock.Lock() - if pushBack { - if pausable, ok := queue.(Pausable); ok { - log.Info("pausing") - pausable.Pause() - } - lock.Unlock() - return data - } - lock.Unlock() - - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - queueShutdown := []func(){} - queueTerminate := []func(){} - terminated := make(chan struct{}) - - tmpDir := t.TempDir() - - queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ - DataDir: tmpDir, - BatchLength: 2, - QueueLength: 20, - Workers: 1, - BoostWorkers: 0, - MaxWorkers: 10, - Name: "test-queue", - }, &testData{}) - assert.NoError(t, err) - - go func() { - queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - queueTerminate = append(queueTerminate, terminate) - }) - close(terminated) - }() - - // Shutdown and Terminate in defer - defer func() { - lock.Lock() - callbacks := make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - lock.Lock() - log.Info("Finally terminating") - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - lock.Unlock() - for _, callback := range callbacks { - callback() - } - }() - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - - err = queue.Push(&test1) - assert.NoError(t, err) - - pausable, ok := queue.(Pausable) - if !assert.True(t, ok) { - return - } - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - pausable.Pause() - paused, _ := pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue is not paused") - return - } - - queue.Push(&test2) - - var result2 *testData - select { - case result2 = <-handleChan: - assert.Fail(t, "handler chan should be empty") - case <-time.After(100 * time.Millisecond): - } - - assert.Nil(t, result2) - - pausable.Resume() - _, resumed := pausable.IsPausedIsResumed() - - select { - case <-resumed: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue should be resumed") - return - } - - select { - case result2 = <-handleChan: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "handler chan should contain test2") - } - - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - // Set pushBack to so that the next handle will result in a Pause - lock.Lock() - pushBack = true - lock.Unlock() - - // Ensure that we're still resumed - _, resumed = pausable.IsPausedIsResumed() - - select { - case <-resumed: - case <-time.After(100 * time.Millisecond): - assert.Fail(t, "Queue is not resumed") - return - } - - // push test1 - queue.Push(&test1) - - // Now as this is handled it should pause - paused, _ = pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-handleChan: - assert.Fail(t, "handler chan should not contain test1") - return - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "queue should be paused") - return - } - - lock.Lock() - pushBack = false - lock.Unlock() - - pausable.Resume() - - _, resumed = pausable.IsPausedIsResumed() - select { - case <-resumed: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "Queue should be resumed") - return - } - - select { - case result1 = <-handleChan: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "handler chan should contain test1") - return - } - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - lock.Lock() - callbacks := make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - queueShutdown = queueShutdown[:0] - lock.Unlock() - // Now shutdown the queue - for _, callback := range callbacks { - callback() - } - - // Wait til it is closed - select { - case <-queue.(*PersistableChannelQueue).closed: - case <-time.After(5 * time.Second): - assert.Fail(t, "queue should close") - return - } - - err = queue.Push(&test1) - assert.NoError(t, err) - err = queue.Push(&test2) - assert.NoError(t, err) - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - return - default: - } - - // terminate the queue - lock.Lock() - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - queueShutdown = queueTerminate[:0] - lock.Unlock() - for _, callback := range callbacks { - callback() - } - - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - return - case <-terminated: - case <-time.After(10 * time.Second): - assert.Fail(t, "Queue should have terminated") - return - } - - lock.Lock() - pushBack = true - lock.Unlock() - - // Reopen queue - terminated = make(chan struct{}) - queue, err = NewPersistableChannelQueue(handle, PersistableChannelQueueConfiguration{ - DataDir: tmpDir, - BatchLength: 1, - QueueLength: 20, - Workers: 1, - BoostWorkers: 0, - MaxWorkers: 10, - Name: "test-queue", - }, &testData{}) - assert.NoError(t, err) - pausable, ok = queue.(Pausable) - if !assert.True(t, ok) { - return - } - - paused, _ = pausable.IsPausedIsResumed() - - go func() { - queue.Run(func(shutdown func()) { - lock.Lock() - defer lock.Unlock() - queueShutdown = append(queueShutdown, shutdown) - }, func(terminate func()) { - lock.Lock() - defer lock.Unlock() - queueTerminate = append(queueTerminate, terminate) - }) - close(terminated) - }() - - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - return - case <-paused: - } - - paused, _ = pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "Queue is not paused") - return - } - - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - return - default: - } - - lock.Lock() - pushBack = false - lock.Unlock() - - pausable.Resume() - _, resumed = pausable.IsPausedIsResumed() - select { - case <-resumed: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "Queue should be resumed") - return - } - - var result3, result4 *testData - - select { - case result3 = <-handleChan: - case <-time.After(1 * time.Second): - assert.Fail(t, "Handler processing should have resumed") - return - } - select { - case result4 = <-handleChan: - case <-time.After(1 * time.Second): - assert.Fail(t, "Handler processing should have resumed") - return - } - if result4.TestString == test1.TestString { - result3, result4 = result4, result3 - } - assert.Equal(t, test1.TestString, result3.TestString) - assert.Equal(t, test1.TestInt, result3.TestInt) - - assert.Equal(t, test2.TestString, result4.TestString) - assert.Equal(t, test2.TestInt, result4.TestInt) - - lock.Lock() - callbacks = make([]func(), len(queueShutdown)) - copy(callbacks, queueShutdown) - queueShutdown = queueShutdown[:0] - lock.Unlock() - // Now shutdown the queue - for _, callback := range callbacks { - callback() - } - - // terminate the queue - lock.Lock() - callbacks = make([]func(), len(queueTerminate)) - copy(callbacks, queueTerminate) - queueShutdown = queueTerminate[:0] - lock.Unlock() - for _, callback := range callbacks { - callback() - } - - select { - case <-time.After(10 * time.Second): - assert.Fail(t, "Queue should have terminated") - return - case <-terminated: - } -} diff --git a/modules/queue/queue_disk_test.go b/modules/queue/queue_disk_test.go deleted file mode 100644 index 8f83abf42..000000000 --- a/modules/queue/queue_disk_test.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "sync" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestLevelQueue(t *testing.T) { - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - assert.True(t, len(data) == 2) - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - var lock sync.Mutex - queueShutdown := []func(){} - queueTerminate := []func(){} - - tmpDir := t.TempDir() - - queue, err := NewLevelQueue(handle, LevelQueueConfiguration{ - ByteFIFOQueueConfiguration: ByteFIFOQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 20, - BatchLength: 2, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 5, - MaxWorkers: 10, - }, - Workers: 1, - }, - DataDir: tmpDir, - }, &testData{}) - assert.NoError(t, err) - - go queue.Run(func(shutdown func()) { - lock.Lock() - queueShutdown = append(queueShutdown, shutdown) - lock.Unlock() - }, func(terminate func()) { - lock.Lock() - queueTerminate = append(queueTerminate, terminate) - lock.Unlock() - }) - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - - err = queue.Push(&test1) - assert.NoError(t, err) - go func() { - err := queue.Push(&test2) - assert.NoError(t, err) - }() - - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - result2 := <-handleChan - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - err = queue.Push(test1) - assert.Error(t, err) - - lock.Lock() - for _, callback := range queueShutdown { - callback() - } - lock.Unlock() - - time.Sleep(200 * time.Millisecond) - err = queue.Push(&test1) - assert.NoError(t, err) - err = queue.Push(&test2) - assert.NoError(t, err) - select { - case <-handleChan: - assert.Fail(t, "Handler processing should have stopped") - default: - } - lock.Lock() - for _, callback := range queueTerminate { - callback() - } - lock.Unlock() - - // Reopen queue - queue, err = NewWrappedQueue(handle, - WrappedQueueConfiguration{ - Underlying: LevelQueueType, - Config: LevelQueueConfiguration{ - ByteFIFOQueueConfiguration: ByteFIFOQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 20, - BatchLength: 2, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 5, - MaxWorkers: 10, - }, - Workers: 1, - }, - DataDir: tmpDir, - }, - }, &testData{}) - assert.NoError(t, err) - - go queue.Run(func(shutdown func()) { - lock.Lock() - queueShutdown = append(queueShutdown, shutdown) - lock.Unlock() - }, func(terminate func()) { - lock.Lock() - queueTerminate = append(queueTerminate, terminate) - lock.Unlock() - }) - - result3 := <-handleChan - assert.Equal(t, test1.TestString, result3.TestString) - assert.Equal(t, test1.TestInt, result3.TestInt) - - result4 := <-handleChan - assert.Equal(t, test2.TestString, result4.TestString) - assert.Equal(t, test2.TestInt, result4.TestInt) - - lock.Lock() - for _, callback := range queueShutdown { - callback() - } - for _, callback := range queueTerminate { - callback() - } - lock.Unlock() -} diff --git a/modules/queue/queue_redis.go b/modules/queue/queue_redis.go deleted file mode 100644 index f8842fea9..000000000 --- a/modules/queue/queue_redis.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - - "code.gitea.io/gitea/modules/graceful" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/nosql" - - "github.com/redis/go-redis/v9" -) - -// RedisQueueType is the type for redis queue -const RedisQueueType Type = "redis" - -// RedisQueueConfiguration is the configuration for the redis queue -type RedisQueueConfiguration struct { - ByteFIFOQueueConfiguration - RedisByteFIFOConfiguration -} - -// RedisQueue redis queue -type RedisQueue struct { - *ByteFIFOQueue -} - -// NewRedisQueue creates single redis or cluster redis queue -func NewRedisQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(RedisQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(RedisQueueConfiguration) - - byteFIFO, err := NewRedisByteFIFO(config.RedisByteFIFOConfiguration) - if err != nil { - return nil, err - } - - byteFIFOQueue, err := NewByteFIFOQueue(RedisQueueType, byteFIFO, handle, config.ByteFIFOQueueConfiguration, exemplar) - if err != nil { - return nil, err - } - - queue := &RedisQueue{ - ByteFIFOQueue: byteFIFOQueue, - } - - queue.qid = GetManager().Add(queue, RedisQueueType, config, exemplar) - - return queue, nil -} - -type redisClient interface { - RPush(ctx context.Context, key string, args ...interface{}) *redis.IntCmd - LPush(ctx context.Context, key string, args ...interface{}) *redis.IntCmd - LPop(ctx context.Context, key string) *redis.StringCmd - LLen(ctx context.Context, key string) *redis.IntCmd - SAdd(ctx context.Context, key string, members ...interface{}) *redis.IntCmd - SRem(ctx context.Context, key string, members ...interface{}) *redis.IntCmd - SIsMember(ctx context.Context, key string, member interface{}) *redis.BoolCmd - Ping(ctx context.Context) *redis.StatusCmd - Close() error -} - -var _ ByteFIFO = &RedisByteFIFO{} - -// RedisByteFIFO represents a ByteFIFO formed from a redisClient -type RedisByteFIFO struct { - client redisClient - - queueName string -} - -// RedisByteFIFOConfiguration is the configuration for the RedisByteFIFO -type RedisByteFIFOConfiguration struct { - ConnectionString string - QueueName string -} - -// NewRedisByteFIFO creates a ByteFIFO formed from a redisClient -func NewRedisByteFIFO(config RedisByteFIFOConfiguration) (*RedisByteFIFO, error) { - fifo := &RedisByteFIFO{ - queueName: config.QueueName, - } - fifo.client = nosql.GetManager().GetRedisClient(config.ConnectionString) - if err := fifo.client.Ping(graceful.GetManager().ShutdownContext()).Err(); err != nil { - return nil, err - } - return fifo, nil -} - -// PushFunc pushes data to the end of the fifo and calls the callback if it is added -func (fifo *RedisByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { - if fn != nil { - if err := fn(); err != nil { - return err - } - } - return fifo.client.RPush(ctx, fifo.queueName, data).Err() -} - -// PushBack pushes data to the top of the fifo -func (fifo *RedisByteFIFO) PushBack(ctx context.Context, data []byte) error { - return fifo.client.LPush(ctx, fifo.queueName, data).Err() -} - -// Pop pops data from the start of the fifo -func (fifo *RedisByteFIFO) Pop(ctx context.Context) ([]byte, error) { - data, err := fifo.client.LPop(ctx, fifo.queueName).Bytes() - if err == nil || err == redis.Nil { - return data, nil - } - return data, err -} - -// Close this fifo -func (fifo *RedisByteFIFO) Close() error { - return fifo.client.Close() -} - -// Len returns the length of the fifo -func (fifo *RedisByteFIFO) Len(ctx context.Context) int64 { - val, err := fifo.client.LLen(ctx, fifo.queueName).Result() - if err != nil { - log.Error("Error whilst getting length of redis queue %s: Error: %v", fifo.queueName, err) - return -1 - } - return val -} - -func init() { - queuesMap[RedisQueueType] = NewRedisQueue -} diff --git a/modules/queue/queue_test.go b/modules/queue/queue_test.go deleted file mode 100644 index 42d34c806..000000000 --- a/modules/queue/queue_test.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "testing" - - "code.gitea.io/gitea/modules/json" - - "github.com/stretchr/testify/assert" -) - -type testData struct { - TestString string - TestInt int -} - -func TestToConfig(t *testing.T) { - cfg := testData{ - TestString: "Config", - TestInt: 10, - } - exemplar := testData{} - - cfg2I, err := toConfig(exemplar, cfg) - assert.NoError(t, err) - cfg2, ok := (cfg2I).(testData) - assert.True(t, ok) - assert.NotEqual(t, cfg2, exemplar) - assert.Equal(t, &cfg, &cfg2) - cfgString, err := json.Marshal(cfg) - assert.NoError(t, err) - - cfg3I, err := toConfig(exemplar, cfgString) - assert.NoError(t, err) - cfg3, ok := (cfg3I).(testData) - assert.True(t, ok) - assert.Equal(t, cfg.TestString, cfg3.TestString) - assert.Equal(t, cfg.TestInt, cfg3.TestInt) - assert.NotEqual(t, cfg3, exemplar) -} diff --git a/modules/queue/queue_wrapped.go b/modules/queue/queue_wrapped.go deleted file mode 100644 index 84d6dd98a..000000000 --- a/modules/queue/queue_wrapped.go +++ /dev/null @@ -1,315 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "fmt" - "sync" - "sync/atomic" - "time" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/util" -) - -// WrappedQueueType is the type for a wrapped delayed starting queue -const WrappedQueueType Type = "wrapped" - -// WrappedQueueConfiguration is the configuration for a WrappedQueue -type WrappedQueueConfiguration struct { - Underlying Type - Timeout time.Duration - MaxAttempts int - Config interface{} - QueueLength int - Name string -} - -type delayedStarter struct { - internal Queue - underlying Type - cfg interface{} - timeout time.Duration - maxAttempts int - name string -} - -// setInternal must be called with the lock locked. -func (q *delayedStarter) setInternal(atShutdown func(func()), handle HandlerFunc, exemplar interface{}) error { - var ctx context.Context - var cancel context.CancelFunc - if q.timeout > 0 { - ctx, cancel = context.WithTimeout(context.Background(), q.timeout) - } else { - ctx, cancel = context.WithCancel(context.Background()) - } - - defer cancel() - // Ensure we also stop at shutdown - atShutdown(cancel) - - i := 1 - for q.internal == nil { - select { - case <-ctx.Done(): - cfg := q.cfg - if s, ok := cfg.([]byte); ok { - cfg = string(s) - } - return fmt.Errorf("timedout creating queue %v with cfg %#v in %s", q.underlying, cfg, q.name) - default: - queue, err := NewQueue(q.underlying, handle, q.cfg, exemplar) - if err == nil { - q.internal = queue - break - } - if err.Error() != "resource temporarily unavailable" { - if bs, ok := q.cfg.([]byte); ok { - log.Warn("[Attempt: %d] Failed to create queue: %v for %s cfg: %s error: %v", i, q.underlying, q.name, string(bs), err) - } else { - log.Warn("[Attempt: %d] Failed to create queue: %v for %s cfg: %#v error: %v", i, q.underlying, q.name, q.cfg, err) - } - } - i++ - if q.maxAttempts > 0 && i > q.maxAttempts { - if bs, ok := q.cfg.([]byte); ok { - return fmt.Errorf("unable to create queue %v for %s with cfg %s by max attempts: error: %w", q.underlying, q.name, string(bs), err) - } - return fmt.Errorf("unable to create queue %v for %s with cfg %#v by max attempts: error: %w", q.underlying, q.name, q.cfg, err) - } - sleepTime := 100 * time.Millisecond - if q.timeout > 0 && q.maxAttempts > 0 { - sleepTime = (q.timeout - 200*time.Millisecond) / time.Duration(q.maxAttempts) - } - t := time.NewTimer(sleepTime) - select { - case <-ctx.Done(): - util.StopTimer(t) - case <-t.C: - } - } - } - return nil -} - -// WrappedQueue wraps a delayed starting queue -type WrappedQueue struct { - delayedStarter - lock sync.Mutex - handle HandlerFunc - exemplar interface{} - channel chan Data - numInQueue int64 -} - -// NewWrappedQueue will attempt to create a queue of the provided type, -// but if there is a problem creating this queue it will instead create -// a WrappedQueue with delayed startup of the queue instead and a -// channel which will be redirected to the queue -func NewWrappedQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(WrappedQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(WrappedQueueConfiguration) - - queue, err := NewQueue(config.Underlying, handle, config.Config, exemplar) - if err == nil { - // Just return the queue there is no need to wrap - return queue, nil - } - if IsErrInvalidConfiguration(err) { - // Retrying ain't gonna make this any better... - return nil, ErrInvalidConfiguration{cfg: cfg} - } - - queue = &WrappedQueue{ - handle: handle, - channel: make(chan Data, config.QueueLength), - exemplar: exemplar, - delayedStarter: delayedStarter{ - cfg: config.Config, - underlying: config.Underlying, - timeout: config.Timeout, - maxAttempts: config.MaxAttempts, - name: config.Name, - }, - } - _ = GetManager().Add(queue, WrappedQueueType, config, exemplar) - return queue, nil -} - -// Name returns the name of the queue -func (q *WrappedQueue) Name() string { - return q.name + "-wrapper" -} - -// Push will push the data to the internal channel checking it against the exemplar -func (q *WrappedQueue) Push(data Data) error { - if !assignableTo(data, q.exemplar) { - return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) - } - atomic.AddInt64(&q.numInQueue, 1) - q.channel <- data - return nil -} - -func (q *WrappedQueue) flushInternalWithContext(ctx context.Context) error { - q.lock.Lock() - if q.internal == nil { - q.lock.Unlock() - return fmt.Errorf("not ready to flush wrapped queue %s yet", q.Name()) - } - q.lock.Unlock() - select { - case <-ctx.Done(): - return ctx.Err() - default: - } - return q.internal.FlushWithContext(ctx) -} - -// Flush flushes the queue and blocks till the queue is empty -func (q *WrappedQueue) Flush(timeout time.Duration) error { - var ctx context.Context - var cancel context.CancelFunc - if timeout > 0 { - ctx, cancel = context.WithTimeout(context.Background(), timeout) - } else { - ctx, cancel = context.WithCancel(context.Background()) - } - defer cancel() - return q.FlushWithContext(ctx) -} - -// FlushWithContext implements the final part of Flushable -func (q *WrappedQueue) FlushWithContext(ctx context.Context) error { - log.Trace("WrappedQueue: %s FlushWithContext", q.Name()) - errChan := make(chan error, 1) - go func() { - errChan <- q.flushInternalWithContext(ctx) - close(errChan) - }() - - select { - case err := <-errChan: - return err - case <-ctx.Done(): - go func() { - <-errChan - }() - return ctx.Err() - } -} - -// IsEmpty checks whether the queue is empty -func (q *WrappedQueue) IsEmpty() bool { - if atomic.LoadInt64(&q.numInQueue) != 0 { - return false - } - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return false - } - return q.internal.IsEmpty() -} - -// Run starts to run the queue and attempts to create the internal queue -func (q *WrappedQueue) Run(atShutdown, atTerminate func(func())) { - log.Debug("WrappedQueue: %s Starting", q.name) - q.lock.Lock() - if q.internal == nil { - err := q.setInternal(atShutdown, q.handle, q.exemplar) - q.lock.Unlock() - if err != nil { - log.Fatal("Unable to set the internal queue for %s Error: %v", q.Name(), err) - return - } - go func() { - for data := range q.channel { - _ = q.internal.Push(data) - atomic.AddInt64(&q.numInQueue, -1) - } - }() - } else { - q.lock.Unlock() - } - - q.internal.Run(atShutdown, atTerminate) - log.Trace("WrappedQueue: %s Done", q.name) -} - -// Shutdown this queue and stop processing -func (q *WrappedQueue) Shutdown() { - log.Trace("WrappedQueue: %s Shutting down", q.name) - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return - } - if shutdownable, ok := q.internal.(Shutdownable); ok { - shutdownable.Shutdown() - } - log.Debug("WrappedQueue: %s Shutdown", q.name) -} - -// Terminate this queue and close the queue -func (q *WrappedQueue) Terminate() { - log.Trace("WrappedQueue: %s Terminating", q.name) - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return - } - if shutdownable, ok := q.internal.(Shutdownable); ok { - shutdownable.Terminate() - } - log.Debug("WrappedQueue: %s Terminated", q.name) -} - -// IsPaused will return if the pool or queue is paused -func (q *WrappedQueue) IsPaused() bool { - q.lock.Lock() - defer q.lock.Unlock() - pausable, ok := q.internal.(Pausable) - return ok && pausable.IsPaused() -} - -// Pause will pause the pool or queue -func (q *WrappedQueue) Pause() { - q.lock.Lock() - defer q.lock.Unlock() - if pausable, ok := q.internal.(Pausable); ok { - pausable.Pause() - } -} - -// Resume will resume the pool or queue -func (q *WrappedQueue) Resume() { - q.lock.Lock() - defer q.lock.Unlock() - if pausable, ok := q.internal.(Pausable); ok { - pausable.Resume() - } -} - -// IsPausedIsResumed will return a bool indicating if the pool or queue is paused and a channel that will be closed when it is resumed -func (q *WrappedQueue) IsPausedIsResumed() (paused, resumed <-chan struct{}) { - q.lock.Lock() - defer q.lock.Unlock() - if pausable, ok := q.internal.(Pausable); ok { - return pausable.IsPausedIsResumed() - } - return context.Background().Done(), closedChan -} - -var closedChan chan struct{} - -func init() { - queuesMap[WrappedQueueType] = NewWrappedQueue - closedChan = make(chan struct{}) - close(closedChan) -} diff --git a/modules/queue/setting.go b/modules/queue/setting.go deleted file mode 100644 index 1e5259fcf..000000000 --- a/modules/queue/setting.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "fmt" - "strings" - - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/setting" -) - -func validType(t string) (Type, error) { - if len(t) == 0 { - return PersistableChannelQueueType, nil - } - for _, typ := range RegisteredTypes() { - if t == string(typ) { - return typ, nil - } - } - return PersistableChannelQueueType, fmt.Errorf("unknown queue type: %s defaulting to %s", t, string(PersistableChannelQueueType)) -} - -func getQueueSettings(name string) (setting.QueueSettings, []byte) { - q := setting.GetQueueSettings(name) - cfg, err := json.Marshal(q) - if err != nil { - log.Error("Unable to marshall generic options: %v Error: %v", q, err) - log.Error("Unable to create queue for %s", name, err) - return q, []byte{} - } - return q, cfg -} - -// CreateQueue for name with provided handler and exemplar -func CreateQueue(name string, handle HandlerFunc, exemplar interface{}) Queue { - q, cfg := getQueueSettings(name) - if len(cfg) == 0 { - return nil - } - - typ, err := validType(q.Type) - if err != nil { - log.Error("Invalid type %s provided for queue named %s defaulting to %s", q.Type, name, string(typ)) - } - - returnable, err := NewQueue(typ, handle, cfg, exemplar) - if q.WrapIfNecessary && err != nil { - log.Warn("Unable to create queue for %s: %v", name, err) - log.Warn("Attempting to create wrapped queue") - returnable, err = NewQueue(WrappedQueueType, handle, WrappedQueueConfiguration{ - Underlying: typ, - Timeout: q.Timeout, - MaxAttempts: q.MaxAttempts, - Config: cfg, - QueueLength: q.QueueLength, - Name: name, - }, exemplar) - } - if err != nil { - log.Error("Unable to create queue for %s: %v", name, err) - return nil - } - - // Sanity check configuration - if q.Workers == 0 && (q.BoostTimeout == 0 || q.BoostWorkers == 0 || q.MaxWorkers == 0) { - log.Warn("Queue: %s is configured to be non-scaling and have no workers\n - this configuration is likely incorrect and could cause Gitea to block", q.Name) - if pausable, ok := returnable.(Pausable); ok { - log.Warn("Queue: %s is being paused to prevent data-loss, add workers manually and unpause.", q.Name) - pausable.Pause() - } - } - - return returnable -} - -// CreateUniqueQueue for name with provided handler and exemplar -func CreateUniqueQueue(name string, handle HandlerFunc, exemplar interface{}) UniqueQueue { - q, cfg := getQueueSettings(name) - if len(cfg) == 0 { - return nil - } - - if len(q.Type) > 0 && q.Type != "dummy" && q.Type != "immediate" && !strings.HasPrefix(q.Type, "unique-") { - q.Type = "unique-" + q.Type - } - - typ, err := validType(q.Type) - if err != nil || typ == PersistableChannelQueueType { - typ = PersistableChannelUniqueQueueType - if err != nil { - log.Error("Invalid type %s provided for queue named %s defaulting to %s", q.Type, name, string(typ)) - } - } - - returnable, err := NewQueue(typ, handle, cfg, exemplar) - if q.WrapIfNecessary && err != nil { - log.Warn("Unable to create unique queue for %s: %v", name, err) - log.Warn("Attempting to create wrapped queue") - returnable, err = NewQueue(WrappedUniqueQueueType, handle, WrappedUniqueQueueConfiguration{ - Underlying: typ, - Timeout: q.Timeout, - MaxAttempts: q.MaxAttempts, - Config: cfg, - QueueLength: q.QueueLength, - }, exemplar) - } - if err != nil { - log.Error("Unable to create unique queue for %s: %v", name, err) - return nil - } - - // Sanity check configuration - if q.Workers == 0 && (q.BoostTimeout == 0 || q.BoostWorkers == 0 || q.MaxWorkers == 0) { - log.Warn("Queue: %s is configured to be non-scaling and have no workers\n - this configuration is likely incorrect and could cause Gitea to block", q.Name) - if pausable, ok := returnable.(Pausable); ok { - log.Warn("Queue: %s is being paused to prevent data-loss, add workers manually and unpause.", q.Name) - pausable.Pause() - } - } - - return returnable.(UniqueQueue) -} diff --git a/modules/queue/testhelper.go b/modules/queue/testhelper.go new file mode 100644 index 000000000..edfa438b1 --- /dev/null +++ b/modules/queue/testhelper.go @@ -0,0 +1,40 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "fmt" + "sync" +) + +// testStateRecorder is used to record state changes for testing, to help debug async behaviors +type testStateRecorder struct { + records []string + mu sync.Mutex +} + +var testRecorder = &testStateRecorder{} + +func (t *testStateRecorder) Record(format string, args ...any) { + t.mu.Lock() + t.records = append(t.records, fmt.Sprintf(format, args...)) + if len(t.records) > 1000 { + t.records = t.records[len(t.records)-1000:] + } + t.mu.Unlock() +} + +func (t *testStateRecorder) Records() []string { + t.mu.Lock() + r := make([]string, len(t.records)) + copy(r, t.records) + t.mu.Unlock() + return r +} + +func (t *testStateRecorder) Reset() { + t.mu.Lock() + t.records = nil + t.mu.Unlock() +} diff --git a/modules/queue/unique_queue.go b/modules/queue/unique_queue.go deleted file mode 100644 index 8f8215c71..000000000 --- a/modules/queue/unique_queue.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "fmt" -) - -// UniqueQueue defines a queue which guarantees only one instance of same -// data is in the queue. Instances with same identity will be -// discarded if there is already one in the line. -// -// This queue is particularly useful for preventing duplicated task -// of same purpose - please note that this does not guarantee that a particular -// task cannot be processed twice or more at the same time. Uniqueness is -// only guaranteed whilst the task is waiting in the queue. -// -// Users of this queue should be careful to push only the identifier of the -// data -type UniqueQueue interface { - Queue - PushFunc(Data, func() error) error - Has(Data) (bool, error) -} - -// ErrAlreadyInQueue is returned when trying to push data to the queue that is already in the queue -var ErrAlreadyInQueue = fmt.Errorf("already in queue") diff --git a/modules/queue/unique_queue_channel.go b/modules/queue/unique_queue_channel.go deleted file mode 100644 index 62c051aa3..000000000 --- a/modules/queue/unique_queue_channel.go +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "fmt" - "runtime/pprof" - "sync" - "time" - - "code.gitea.io/gitea/modules/container" - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" -) - -// ChannelUniqueQueueType is the type for channel queue -const ChannelUniqueQueueType Type = "unique-channel" - -// ChannelUniqueQueueConfiguration is the configuration for a ChannelUniqueQueue -type ChannelUniqueQueueConfiguration ChannelQueueConfiguration - -// ChannelUniqueQueue implements UniqueQueue -// -// It is basically a thin wrapper around a WorkerPool but keeps a store of -// what has been pushed within a table. -// -// Please note that this Queue does not guarantee that a particular -// task cannot be processed twice or more at the same time. Uniqueness is -// only guaranteed whilst the task is waiting in the queue. -type ChannelUniqueQueue struct { - *WorkerPool - lock sync.Mutex - table container.Set[string] - shutdownCtx context.Context - shutdownCtxCancel context.CancelFunc - terminateCtx context.Context - terminateCtxCancel context.CancelFunc - exemplar interface{} - workers int - name string -} - -// NewChannelUniqueQueue create a memory channel queue -func NewChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(ChannelUniqueQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(ChannelUniqueQueueConfiguration) - if config.BatchLength == 0 { - config.BatchLength = 1 - } - - terminateCtx, terminateCtxCancel := context.WithCancel(context.Background()) - shutdownCtx, shutdownCtxCancel := context.WithCancel(terminateCtx) - - queue := &ChannelUniqueQueue{ - table: make(container.Set[string]), - shutdownCtx: shutdownCtx, - shutdownCtxCancel: shutdownCtxCancel, - terminateCtx: terminateCtx, - terminateCtxCancel: terminateCtxCancel, - exemplar: exemplar, - workers: config.Workers, - name: config.Name, - } - queue.WorkerPool = NewWorkerPool(func(data ...Data) (unhandled []Data) { - for _, datum := range data { - // No error is possible here because PushFunc ensures that this can be marshalled - bs, _ := json.Marshal(datum) - - queue.lock.Lock() - queue.table.Remove(string(bs)) - queue.lock.Unlock() - - if u := handle(datum); u != nil { - if queue.IsPaused() { - // We can only pushback to the channel if we're paused. - go func() { - if err := queue.Push(u[0]); err != nil { - log.Error("Unable to push back to queue %d. Error: %v", queue.qid, err) - } - }() - } else { - unhandled = append(unhandled, u...) - } - } - } - return unhandled - }, config.WorkerPoolConfiguration) - - queue.qid = GetManager().Add(queue, ChannelUniqueQueueType, config, exemplar) - return queue, nil -} - -// Run starts to run the queue -func (q *ChannelUniqueQueue) Run(atShutdown, atTerminate func(func())) { - pprof.SetGoroutineLabels(q.baseCtx) - atShutdown(q.Shutdown) - atTerminate(q.Terminate) - log.Debug("ChannelUniqueQueue: %s Starting", q.name) - _ = q.AddWorkers(q.workers, 0) -} - -// Push will push data into the queue if the data is not already in the queue -func (q *ChannelUniqueQueue) Push(data Data) error { - return q.PushFunc(data, nil) -} - -// PushFunc will push data into the queue -func (q *ChannelUniqueQueue) PushFunc(data Data, fn func() error) error { - if !assignableTo(data, q.exemplar) { - return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in queue: %s", data, q.exemplar, q.name) - } - - bs, err := json.Marshal(data) - if err != nil { - return err - } - q.lock.Lock() - locked := true - defer func() { - if locked { - q.lock.Unlock() - } - }() - if !q.table.Add(string(bs)) { - return ErrAlreadyInQueue - } - // FIXME: We probably need to implement some sort of limit here - // If the downstream queue blocks this table will grow without limit - if fn != nil { - err := fn() - if err != nil { - q.table.Remove(string(bs)) - return err - } - } - locked = false - q.lock.Unlock() - q.WorkerPool.Push(data) - return nil -} - -// Has checks if the data is in the queue -func (q *ChannelUniqueQueue) Has(data Data) (bool, error) { - bs, err := json.Marshal(data) - if err != nil { - return false, err - } - - q.lock.Lock() - defer q.lock.Unlock() - return q.table.Contains(string(bs)), nil -} - -// Flush flushes the channel with a timeout - the Flush worker will be registered as a flush worker with the manager -func (q *ChannelUniqueQueue) Flush(timeout time.Duration) error { - if q.IsPaused() { - return nil - } - ctx, cancel := q.commonRegisterWorkers(1, timeout, true) - defer cancel() - return q.FlushWithContext(ctx) -} - -// Shutdown processing from this queue -func (q *ChannelUniqueQueue) Shutdown() { - log.Trace("ChannelUniqueQueue: %s Shutting down", q.name) - select { - case <-q.shutdownCtx.Done(): - return - default: - } - go func() { - log.Trace("ChannelUniqueQueue: %s Flushing", q.name) - if err := q.FlushWithContext(q.terminateCtx); err != nil { - if !q.IsEmpty() { - log.Warn("ChannelUniqueQueue: %s Terminated before completed flushing", q.name) - } - return - } - log.Debug("ChannelUniqueQueue: %s Flushed", q.name) - }() - q.shutdownCtxCancel() - log.Debug("ChannelUniqueQueue: %s Shutdown", q.name) -} - -// Terminate this queue and close the queue -func (q *ChannelUniqueQueue) Terminate() { - log.Trace("ChannelUniqueQueue: %s Terminating", q.name) - q.Shutdown() - select { - case <-q.terminateCtx.Done(): - return - default: - } - q.terminateCtxCancel() - q.baseCtxFinished() - log.Debug("ChannelUniqueQueue: %s Terminated", q.name) -} - -// Name returns the name of this queue -func (q *ChannelUniqueQueue) Name() string { - return q.name -} - -func init() { - queuesMap[ChannelUniqueQueueType] = NewChannelUniqueQueue -} diff --git a/modules/queue/unique_queue_channel_test.go b/modules/queue/unique_queue_channel_test.go deleted file mode 100644 index 824015b83..000000000 --- a/modules/queue/unique_queue_channel_test.go +++ /dev/null @@ -1,258 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "sync" - "testing" - "time" - - "code.gitea.io/gitea/modules/log" - - "github.com/stretchr/testify/assert" -) - -func TestChannelUniqueQueue(t *testing.T) { - _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - nilFn := func(_ func()) {} - - queue, err := NewChannelUniqueQueue(handle, - ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 0, - MaxWorkers: 10, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 5, - Name: "TestChannelQueue", - }, - Workers: 0, - }, &testData{}) - assert.NoError(t, err) - - assert.Equal(t, queue.(*ChannelUniqueQueue).WorkerPool.boostWorkers, 5) - - go queue.Run(nilFn, nilFn) - - test1 := testData{"A", 1} - go queue.Push(&test1) - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - err = queue.Push(test1) - assert.Error(t, err) -} - -func TestChannelUniqueQueue_Batch(t *testing.T) { - _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) - - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - - nilFn := func(_ func()) {} - - queue, err := NewChannelUniqueQueue(handle, - ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 20, - BatchLength: 2, - BlockTimeout: 0, - BoostTimeout: 0, - BoostWorkers: 0, - MaxWorkers: 10, - }, - Workers: 1, - }, &testData{}) - assert.NoError(t, err) - - go queue.Run(nilFn, nilFn) - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - - queue.Push(&test1) - go queue.Push(&test2) - - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - result2 := <-handleChan - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - err = queue.Push(test1) - assert.Error(t, err) -} - -func TestChannelUniqueQueue_Pause(t *testing.T) { - _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) - - lock := sync.Mutex{} - var queue Queue - var err error - pushBack := false - handleChan := make(chan *testData) - handle := func(data ...Data) []Data { - lock.Lock() - if pushBack { - if pausable, ok := queue.(Pausable); ok { - pausable.Pause() - } - pushBack = false - lock.Unlock() - return data - } - lock.Unlock() - - for _, datum := range data { - testDatum := datum.(*testData) - handleChan <- testDatum - } - return nil - } - nilFn := func(_ func()) {} - - queue, err = NewChannelUniqueQueue(handle, - ChannelQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: 20, - BatchLength: 1, - BlockTimeout: 0, - BoostTimeout: 0, - BoostWorkers: 0, - MaxWorkers: 10, - }, - Workers: 1, - }, &testData{}) - assert.NoError(t, err) - - go queue.Run(nilFn, nilFn) - - test1 := testData{"A", 1} - test2 := testData{"B", 2} - queue.Push(&test1) - - pausable, ok := queue.(Pausable) - if !assert.True(t, ok) { - return - } - result1 := <-handleChan - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) - - pausable.Pause() - - paused, resumed := pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-resumed: - assert.Fail(t, "Queue should not be resumed") - return - default: - assert.Fail(t, "Queue is not paused") - return - } - - queue.Push(&test2) - - var result2 *testData - select { - case result2 = <-handleChan: - assert.Fail(t, "handler chan should be empty") - case <-time.After(100 * time.Millisecond): - } - - assert.Nil(t, result2) - - pausable.Resume() - - select { - case <-resumed: - default: - assert.Fail(t, "Queue should be resumed") - } - - select { - case result2 = <-handleChan: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "handler chan should contain test2") - } - - assert.Equal(t, test2.TestString, result2.TestString) - assert.Equal(t, test2.TestInt, result2.TestInt) - - lock.Lock() - pushBack = true - lock.Unlock() - - paused, resumed = pausable.IsPausedIsResumed() - - select { - case <-paused: - assert.Fail(t, "Queue should not be paused") - return - case <-resumed: - default: - assert.Fail(t, "Queue is not resumed") - return - } - - queue.Push(&test1) - - select { - case <-paused: - case <-handleChan: - assert.Fail(t, "handler chan should not contain test1") - return - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "queue should be paused") - return - } - - paused, resumed = pausable.IsPausedIsResumed() - - select { - case <-paused: - case <-resumed: - assert.Fail(t, "Queue should not be resumed") - return - default: - assert.Fail(t, "Queue is not paused") - return - } - - pausable.Resume() - - select { - case <-resumed: - default: - assert.Fail(t, "Queue should be resumed") - } - - select { - case result1 = <-handleChan: - case <-time.After(500 * time.Millisecond): - assert.Fail(t, "handler chan should contain test1") - } - assert.Equal(t, test1.TestString, result1.TestString) - assert.Equal(t, test1.TestInt, result1.TestInt) -} diff --git a/modules/queue/unique_queue_disk.go b/modules/queue/unique_queue_disk.go deleted file mode 100644 index 406f64784..000000000 --- a/modules/queue/unique_queue_disk.go +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - - "code.gitea.io/gitea/modules/nosql" - - "gitea.com/lunny/levelqueue" -) - -// LevelUniqueQueueType is the type for level queue -const LevelUniqueQueueType Type = "unique-level" - -// LevelUniqueQueueConfiguration is the configuration for a LevelUniqueQueue -type LevelUniqueQueueConfiguration struct { - ByteFIFOQueueConfiguration - DataDir string - ConnectionString string - QueueName string -} - -// LevelUniqueQueue implements a disk library queue -type LevelUniqueQueue struct { - *ByteFIFOUniqueQueue -} - -// NewLevelUniqueQueue creates a ledis local queue -// -// Please note that this Queue does not guarantee that a particular -// task cannot be processed twice or more at the same time. Uniqueness is -// only guaranteed whilst the task is waiting in the queue. -func NewLevelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(LevelUniqueQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(LevelUniqueQueueConfiguration) - - if len(config.ConnectionString) == 0 { - config.ConnectionString = config.DataDir - } - config.WaitOnEmpty = true - - byteFIFO, err := NewLevelUniqueQueueByteFIFO(config.ConnectionString, config.QueueName) - if err != nil { - return nil, err - } - - byteFIFOQueue, err := NewByteFIFOUniqueQueue(LevelUniqueQueueType, byteFIFO, handle, config.ByteFIFOQueueConfiguration, exemplar) - if err != nil { - return nil, err - } - - queue := &LevelUniqueQueue{ - ByteFIFOUniqueQueue: byteFIFOQueue, - } - queue.qid = GetManager().Add(queue, LevelUniqueQueueType, config, exemplar) - return queue, nil -} - -var _ UniqueByteFIFO = &LevelUniqueQueueByteFIFO{} - -// LevelUniqueQueueByteFIFO represents a ByteFIFO formed from a LevelUniqueQueue -type LevelUniqueQueueByteFIFO struct { - internal *levelqueue.UniqueQueue - connection string -} - -// NewLevelUniqueQueueByteFIFO creates a new ByteFIFO formed from a LevelUniqueQueue -func NewLevelUniqueQueueByteFIFO(connection, prefix string) (*LevelUniqueQueueByteFIFO, error) { - db, err := nosql.GetManager().GetLevelDB(connection) - if err != nil { - return nil, err - } - - internal, err := levelqueue.NewUniqueQueue(db, []byte(prefix), []byte(prefix+"-unique"), false) - if err != nil { - return nil, err - } - - return &LevelUniqueQueueByteFIFO{ - connection: connection, - internal: internal, - }, nil -} - -// PushFunc pushes data to the end of the fifo and calls the callback if it is added -func (fifo *LevelUniqueQueueByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { - return fifo.internal.LPushFunc(data, fn) -} - -// PushBack pushes data to the top of the fifo -func (fifo *LevelUniqueQueueByteFIFO) PushBack(ctx context.Context, data []byte) error { - return fifo.internal.RPush(data) -} - -// Pop pops data from the start of the fifo -func (fifo *LevelUniqueQueueByteFIFO) Pop(ctx context.Context) ([]byte, error) { - data, err := fifo.internal.RPop() - if err != nil && err != levelqueue.ErrNotFound { - return nil, err - } - return data, nil -} - -// Len returns the length of the fifo -func (fifo *LevelUniqueQueueByteFIFO) Len(ctx context.Context) int64 { - return fifo.internal.Len() -} - -// Has returns whether the fifo contains this data -func (fifo *LevelUniqueQueueByteFIFO) Has(ctx context.Context, data []byte) (bool, error) { - return fifo.internal.Has(data) -} - -// Close this fifo -func (fifo *LevelUniqueQueueByteFIFO) Close() error { - err := fifo.internal.Close() - _ = nosql.GetManager().CloseLevelDB(fifo.connection) - return err -} - -func init() { - queuesMap[LevelUniqueQueueType] = NewLevelUniqueQueue -} diff --git a/modules/queue/unique_queue_disk_channel.go b/modules/queue/unique_queue_disk_channel.go deleted file mode 100644 index cc8a807c6..000000000 --- a/modules/queue/unique_queue_disk_channel.go +++ /dev/null @@ -1,336 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "runtime/pprof" - "sync" - "time" - - "code.gitea.io/gitea/modules/log" -) - -// PersistableChannelUniqueQueueType is the type for persistable queue -const PersistableChannelUniqueQueueType Type = "unique-persistable-channel" - -// PersistableChannelUniqueQueueConfiguration is the configuration for a PersistableChannelUniqueQueue -type PersistableChannelUniqueQueueConfiguration struct { - Name string - DataDir string - BatchLength int - QueueLength int - Timeout time.Duration - MaxAttempts int - Workers int - MaxWorkers int - BlockTimeout time.Duration - BoostTimeout time.Duration - BoostWorkers int -} - -// PersistableChannelUniqueQueue wraps a channel queue and level queue together -// -// Please note that this Queue does not guarantee that a particular -// task cannot be processed twice or more at the same time. Uniqueness is -// only guaranteed whilst the task is waiting in the queue. -type PersistableChannelUniqueQueue struct { - channelQueue *ChannelUniqueQueue - delayedStarter - lock sync.Mutex - closed chan struct{} -} - -// NewPersistableChannelUniqueQueue creates a wrapped batched channel queue with persistable level queue backend when shutting down -// This differs from a wrapped queue in that the persistent queue is only used to persist at shutdown/terminate -func NewPersistableChannelUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(PersistableChannelUniqueQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(PersistableChannelUniqueQueueConfiguration) - - queue := &PersistableChannelUniqueQueue{ - closed: make(chan struct{}), - } - - wrappedHandle := func(data ...Data) (failed []Data) { - for _, unhandled := range handle(data...) { - if fail := queue.PushBack(unhandled); fail != nil { - failed = append(failed, fail) - } - } - return failed - } - - channelUniqueQueue, err := NewChannelUniqueQueue(wrappedHandle, ChannelUniqueQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: config.QueueLength, - BatchLength: config.BatchLength, - BlockTimeout: config.BlockTimeout, - BoostTimeout: config.BoostTimeout, - BoostWorkers: config.BoostWorkers, - MaxWorkers: config.MaxWorkers, - Name: config.Name + "-channel", - }, - Workers: config.Workers, - }, exemplar) - if err != nil { - return nil, err - } - - // the level backend only needs temporary workers to catch up with the previously dropped work - levelCfg := LevelUniqueQueueConfiguration{ - ByteFIFOQueueConfiguration: ByteFIFOQueueConfiguration{ - WorkerPoolConfiguration: WorkerPoolConfiguration{ - QueueLength: config.QueueLength, - BatchLength: config.BatchLength, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 1, - MaxWorkers: 5, - Name: config.Name + "-level", - }, - Workers: 0, - }, - DataDir: config.DataDir, - QueueName: config.Name + "-level", - } - - queue.channelQueue = channelUniqueQueue.(*ChannelUniqueQueue) - - levelQueue, err := NewLevelUniqueQueue(func(data ...Data) []Data { - for _, datum := range data { - err := queue.Push(datum) - if err != nil && err != ErrAlreadyInQueue { - log.Error("Unable push to channelled queue: %v", err) - } - } - return nil - }, levelCfg, exemplar) - if err == nil { - queue.delayedStarter = delayedStarter{ - internal: levelQueue.(*LevelUniqueQueue), - name: config.Name, - } - - _ = GetManager().Add(queue, PersistableChannelUniqueQueueType, config, exemplar) - return queue, nil - } - if IsErrInvalidConfiguration(err) { - // Retrying ain't gonna make this any better... - return nil, ErrInvalidConfiguration{cfg: cfg} - } - - queue.delayedStarter = delayedStarter{ - cfg: levelCfg, - underlying: LevelUniqueQueueType, - timeout: config.Timeout, - maxAttempts: config.MaxAttempts, - name: config.Name, - } - _ = GetManager().Add(queue, PersistableChannelUniqueQueueType, config, exemplar) - return queue, nil -} - -// Name returns the name of this queue -func (q *PersistableChannelUniqueQueue) Name() string { - return q.delayedStarter.name -} - -// Push will push the indexer data to queue -func (q *PersistableChannelUniqueQueue) Push(data Data) error { - return q.PushFunc(data, nil) -} - -// PushFunc will push the indexer data to queue -func (q *PersistableChannelUniqueQueue) PushFunc(data Data, fn func() error) error { - select { - case <-q.closed: - return q.internal.(UniqueQueue).PushFunc(data, fn) - default: - return q.channelQueue.PushFunc(data, fn) - } -} - -// PushBack will push the indexer data to queue -func (q *PersistableChannelUniqueQueue) PushBack(data Data) error { - select { - case <-q.closed: - if pbr, ok := q.internal.(PushBackable); ok { - return pbr.PushBack(data) - } - return q.internal.Push(data) - default: - return q.channelQueue.Push(data) - } -} - -// Has will test if the queue has the data -func (q *PersistableChannelUniqueQueue) Has(data Data) (bool, error) { - // This is more difficult... - has, err := q.channelQueue.Has(data) - if err != nil || has { - return has, err - } - q.lock.Lock() - defer q.lock.Unlock() - if q.internal == nil { - return false, nil - } - return q.internal.(UniqueQueue).Has(data) -} - -// Run starts to run the queue -func (q *PersistableChannelUniqueQueue) Run(atShutdown, atTerminate func(func())) { - pprof.SetGoroutineLabels(q.channelQueue.baseCtx) - log.Debug("PersistableChannelUniqueQueue: %s Starting", q.delayedStarter.name) - - q.lock.Lock() - if q.internal == nil { - err := q.setInternal(atShutdown, func(data ...Data) []Data { - for _, datum := range data { - err := q.Push(datum) - if err != nil && err != ErrAlreadyInQueue { - log.Error("Unable push to channelled queue: %v", err) - } - } - return nil - }, q.channelQueue.exemplar) - q.lock.Unlock() - if err != nil { - log.Fatal("Unable to create internal queue for %s Error: %v", q.Name(), err) - return - } - } else { - q.lock.Unlock() - } - atShutdown(q.Shutdown) - atTerminate(q.Terminate) - _ = q.channelQueue.AddWorkers(q.channelQueue.workers, 0) - - if luq, ok := q.internal.(*LevelUniqueQueue); ok && !luq.IsEmpty() { - // Just run the level queue - we shut it down once it's flushed - go luq.Run(func(_ func()) {}, func(_ func()) {}) - go func() { - _ = luq.Flush(0) - for !luq.IsEmpty() { - _ = luq.Flush(0) - select { - case <-time.After(100 * time.Millisecond): - case <-luq.shutdownCtx.Done(): - if luq.byteFIFO.Len(luq.terminateCtx) > 0 { - log.Warn("LevelUniqueQueue: %s shut down before completely flushed", luq.Name()) - } - return - } - } - log.Debug("LevelUniqueQueue: %s flushed so shutting down", luq.Name()) - luq.Shutdown() - GetManager().Remove(luq.qid) - }() - } else { - log.Debug("PersistableChannelUniqueQueue: %s Skipping running the empty level queue", q.delayedStarter.name) - _ = q.internal.Flush(0) - q.internal.(*LevelUniqueQueue).Shutdown() - GetManager().Remove(q.internal.(*LevelUniqueQueue).qid) - } -} - -// Flush flushes the queue -func (q *PersistableChannelUniqueQueue) Flush(timeout time.Duration) error { - return q.channelQueue.Flush(timeout) -} - -// FlushWithContext flushes the queue -func (q *PersistableChannelUniqueQueue) FlushWithContext(ctx context.Context) error { - return q.channelQueue.FlushWithContext(ctx) -} - -// IsEmpty checks if a queue is empty -func (q *PersistableChannelUniqueQueue) IsEmpty() bool { - return q.channelQueue.IsEmpty() -} - -// IsPaused will return if the pool or queue is paused -func (q *PersistableChannelUniqueQueue) IsPaused() bool { - return q.channelQueue.IsPaused() -} - -// Pause will pause the pool or queue -func (q *PersistableChannelUniqueQueue) Pause() { - q.channelQueue.Pause() -} - -// Resume will resume the pool or queue -func (q *PersistableChannelUniqueQueue) Resume() { - q.channelQueue.Resume() -} - -// IsPausedIsResumed will return a bool indicating if the pool or queue is paused and a channel that will be closed when it is resumed -func (q *PersistableChannelUniqueQueue) IsPausedIsResumed() (paused, resumed <-chan struct{}) { - return q.channelQueue.IsPausedIsResumed() -} - -// Shutdown processing this queue -func (q *PersistableChannelUniqueQueue) Shutdown() { - log.Trace("PersistableChannelUniqueQueue: %s Shutting down", q.delayedStarter.name) - q.lock.Lock() - select { - case <-q.closed: - q.lock.Unlock() - return - default: - if q.internal != nil { - q.internal.(*LevelUniqueQueue).Shutdown() - } - close(q.closed) - q.lock.Unlock() - } - - log.Trace("PersistableChannelUniqueQueue: %s Cancelling pools", q.delayedStarter.name) - q.internal.(*LevelUniqueQueue).baseCtxCancel() - q.channelQueue.baseCtxCancel() - log.Trace("PersistableChannelUniqueQueue: %s Waiting til done", q.delayedStarter.name) - q.channelQueue.Wait() - q.internal.(*LevelUniqueQueue).Wait() - // Redirect all remaining data in the chan to the internal channel - close(q.channelQueue.dataChan) - log.Trace("PersistableChannelUniqueQueue: %s Redirecting remaining data", q.delayedStarter.name) - countOK, countLost := 0, 0 - for data := range q.channelQueue.dataChan { - err := q.internal.(*LevelUniqueQueue).Push(data) - if err != nil { - log.Error("PersistableChannelUniqueQueue: %s Unable redirect %v due to: %v", q.delayedStarter.name, data, err) - countLost++ - } else { - countOK++ - } - } - if countLost > 0 { - log.Warn("PersistableChannelUniqueQueue: %s %d will be restored on restart, %d lost", q.delayedStarter.name, countOK, countLost) - } else if countOK > 0 { - log.Warn("PersistableChannelUniqueQueue: %s %d will be restored on restart", q.delayedStarter.name, countOK) - } - log.Trace("PersistableChannelUniqueQueue: %s Done Redirecting remaining data", q.delayedStarter.name) - - log.Debug("PersistableChannelUniqueQueue: %s Shutdown", q.delayedStarter.name) -} - -// Terminate this queue and close the queue -func (q *PersistableChannelUniqueQueue) Terminate() { - log.Trace("PersistableChannelUniqueQueue: %s Terminating", q.delayedStarter.name) - q.Shutdown() - q.lock.Lock() - defer q.lock.Unlock() - if q.internal != nil { - q.internal.(*LevelUniqueQueue).Terminate() - } - q.channelQueue.baseCtxFinished() - log.Debug("PersistableChannelUniqueQueue: %s Terminated", q.delayedStarter.name) -} - -func init() { - queuesMap[PersistableChannelUniqueQueueType] = NewPersistableChannelUniqueQueue -} diff --git a/modules/queue/unique_queue_disk_channel_test.go b/modules/queue/unique_queue_disk_channel_test.go deleted file mode 100644 index 11a1d4b88..000000000 --- a/modules/queue/unique_queue_disk_channel_test.go +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright 2023 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "strconv" - "sync" - "sync/atomic" - "testing" - "time" - - "code.gitea.io/gitea/modules/log" - - "github.com/stretchr/testify/assert" -) - -func TestPersistableChannelUniqueQueue(t *testing.T) { - // Create a temporary directory for the queue - tmpDir := t.TempDir() - _ = log.NewLogger(1000, "console", "console", `{"level":"warn","stacktracelevel":"NONE","stderr":true}`) - - // Common function to create the Queue - newQueue := func(name string, handle func(data ...Data) []Data) Queue { - q, err := NewPersistableChannelUniqueQueue(handle, - PersistableChannelUniqueQueueConfiguration{ - Name: name, - DataDir: tmpDir, - QueueLength: 200, - MaxWorkers: 1, - BlockTimeout: 1 * time.Second, - BoostTimeout: 5 * time.Minute, - BoostWorkers: 1, - Workers: 0, - }, "task-0") - assert.NoError(t, err) - return q - } - - // runs the provided queue and provides some timer function - type channels struct { - readyForShutdown chan struct{} // closed when shutdown functions have been assigned - readyForTerminate chan struct{} // closed when terminate functions have been assigned - signalShutdown chan struct{} // Should close to signal shutdown - doneShutdown chan struct{} // closed when shutdown function is done - queueTerminate []func() // list of atTerminate functions to call atTerminate - need to be accessed with lock - } - runQueue := func(q Queue, lock *sync.Mutex) *channels { - chans := &channels{ - readyForShutdown: make(chan struct{}), - readyForTerminate: make(chan struct{}), - signalShutdown: make(chan struct{}), - doneShutdown: make(chan struct{}), - } - go q.Run(func(atShutdown func()) { - go func() { - lock.Lock() - select { - case <-chans.readyForShutdown: - default: - close(chans.readyForShutdown) - } - lock.Unlock() - <-chans.signalShutdown - atShutdown() - close(chans.doneShutdown) - }() - }, func(atTerminate func()) { - lock.Lock() - defer lock.Unlock() - select { - case <-chans.readyForTerminate: - default: - close(chans.readyForTerminate) - } - chans.queueTerminate = append(chans.queueTerminate, atTerminate) - }) - - return chans - } - - // call to shutdown and terminate the queue associated with the channels - doTerminate := func(chans *channels, lock *sync.Mutex) { - <-chans.readyForTerminate - - lock.Lock() - callbacks := []func(){} - callbacks = append(callbacks, chans.queueTerminate...) - lock.Unlock() - - for _, callback := range callbacks { - callback() - } - } - - mapLock := sync.Mutex{} - executedInitial := map[string][]string{} - hasInitial := map[string][]string{} - - fillQueue := func(name string, done chan int64) { - t.Run("Initial Filling: "+name, func(t *testing.T) { - lock := sync.Mutex{} - - startAt100Queued := make(chan struct{}) - stopAt20Shutdown := make(chan struct{}) // stop and shutdown at the 20th item - - handle := func(data ...Data) []Data { - <-startAt100Queued - for _, datum := range data { - s := datum.(string) - mapLock.Lock() - executedInitial[name] = append(executedInitial[name], s) - mapLock.Unlock() - if s == "task-20" { - close(stopAt20Shutdown) - } - } - return nil - } - - q := newQueue(name, handle) - - // add 100 tasks to the queue - for i := 0; i < 100; i++ { - _ = q.Push("task-" + strconv.Itoa(i)) - } - close(startAt100Queued) - - chans := runQueue(q, &lock) - - <-chans.readyForShutdown - <-stopAt20Shutdown - close(chans.signalShutdown) - <-chans.doneShutdown - _ = q.Push("final") - - // check which tasks are still in the queue - for i := 0; i < 100; i++ { - if has, _ := q.(UniqueQueue).Has("task-" + strconv.Itoa(i)); has { - mapLock.Lock() - hasInitial[name] = append(hasInitial[name], "task-"+strconv.Itoa(i)) - mapLock.Unlock() - } - } - if has, _ := q.(UniqueQueue).Has("final"); has { - mapLock.Lock() - hasInitial[name] = append(hasInitial[name], "final") - mapLock.Unlock() - } else { - assert.Fail(t, "UnqueQueue %s should have \"final\"", name) - } - doTerminate(chans, &lock) - mapLock.Lock() - assert.Equal(t, 101, len(executedInitial[name])+len(hasInitial[name])) - mapLock.Unlock() - }) - mapLock.Lock() - count := int64(len(hasInitial[name])) - mapLock.Unlock() - done <- count - close(done) - } - - hasQueueAChan := make(chan int64) - hasQueueBChan := make(chan int64) - - go fillQueue("QueueA", hasQueueAChan) - go fillQueue("QueueB", hasQueueBChan) - - hasA := <-hasQueueAChan - hasB := <-hasQueueBChan - - executedEmpty := map[string][]string{} - hasEmpty := map[string][]string{} - emptyQueue := func(name string, numInQueue int64, done chan struct{}) { - t.Run("Empty Queue: "+name, func(t *testing.T) { - lock := sync.Mutex{} - stop := make(chan struct{}) - - // collect the tasks that have been executed - atomicCount := int64(0) - handle := func(data ...Data) []Data { - lock.Lock() - for _, datum := range data { - mapLock.Lock() - executedEmpty[name] = append(executedEmpty[name], datum.(string)) - mapLock.Unlock() - count := atomic.AddInt64(&atomicCount, 1) - if count >= numInQueue { - close(stop) - } - } - lock.Unlock() - return nil - } - - q := newQueue(name, handle) - chans := runQueue(q, &lock) - - <-chans.readyForShutdown - <-stop - close(chans.signalShutdown) - <-chans.doneShutdown - - // check which tasks are still in the queue - for i := 0; i < 100; i++ { - if has, _ := q.(UniqueQueue).Has("task-" + strconv.Itoa(i)); has { - mapLock.Lock() - hasEmpty[name] = append(hasEmpty[name], "task-"+strconv.Itoa(i)) - mapLock.Unlock() - } - } - doTerminate(chans, &lock) - - mapLock.Lock() - assert.Equal(t, 101, len(executedInitial[name])+len(executedEmpty[name])) - assert.Empty(t, hasEmpty[name]) - mapLock.Unlock() - }) - close(done) - } - - doneA := make(chan struct{}) - doneB := make(chan struct{}) - - go emptyQueue("QueueA", hasA, doneA) - go emptyQueue("QueueB", hasB, doneB) - - <-doneA - <-doneB - - mapLock.Lock() - t.Logf("TestPersistableChannelUniqueQueue executedInitiallyA=%v, executedInitiallyB=%v, executedToEmptyA=%v, executedToEmptyB=%v", - len(executedInitial["QueueA"]), len(executedInitial["QueueB"]), len(executedEmpty["QueueA"]), len(executedEmpty["QueueB"])) - - // reset and rerun - executedInitial = map[string][]string{} - hasInitial = map[string][]string{} - executedEmpty = map[string][]string{} - hasEmpty = map[string][]string{} - mapLock.Unlock() - - hasQueueAChan = make(chan int64) - hasQueueBChan = make(chan int64) - - go fillQueue("QueueA", hasQueueAChan) - go fillQueue("QueueB", hasQueueBChan) - - hasA = <-hasQueueAChan - hasB = <-hasQueueBChan - - doneA = make(chan struct{}) - doneB = make(chan struct{}) - - go emptyQueue("QueueA", hasA, doneA) - go emptyQueue("QueueB", hasB, doneB) - - <-doneA - <-doneB - - mapLock.Lock() - t.Logf("TestPersistableChannelUniqueQueue executedInitiallyA=%v, executedInitiallyB=%v, executedToEmptyA=%v, executedToEmptyB=%v", - len(executedInitial["QueueA"]), len(executedInitial["QueueB"]), len(executedEmpty["QueueA"]), len(executedEmpty["QueueB"])) - mapLock.Unlock() -} diff --git a/modules/queue/unique_queue_redis.go b/modules/queue/unique_queue_redis.go deleted file mode 100644 index ae1df08eb..000000000 --- a/modules/queue/unique_queue_redis.go +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - - "github.com/redis/go-redis/v9" -) - -// RedisUniqueQueueType is the type for redis queue -const RedisUniqueQueueType Type = "unique-redis" - -// RedisUniqueQueue redis queue -type RedisUniqueQueue struct { - *ByteFIFOUniqueQueue -} - -// RedisUniqueQueueConfiguration is the configuration for the redis queue -type RedisUniqueQueueConfiguration struct { - ByteFIFOQueueConfiguration - RedisUniqueByteFIFOConfiguration -} - -// NewRedisUniqueQueue creates single redis or cluster redis queue. -// -// Please note that this Queue does not guarantee that a particular -// task cannot be processed twice or more at the same time. Uniqueness is -// only guaranteed whilst the task is waiting in the queue. -func NewRedisUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(RedisUniqueQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(RedisUniqueQueueConfiguration) - - byteFIFO, err := NewRedisUniqueByteFIFO(config.RedisUniqueByteFIFOConfiguration) - if err != nil { - return nil, err - } - - if len(byteFIFO.setName) == 0 { - byteFIFO.setName = byteFIFO.queueName + "_unique" - } - - byteFIFOQueue, err := NewByteFIFOUniqueQueue(RedisUniqueQueueType, byteFIFO, handle, config.ByteFIFOQueueConfiguration, exemplar) - if err != nil { - return nil, err - } - - queue := &RedisUniqueQueue{ - ByteFIFOUniqueQueue: byteFIFOQueue, - } - - queue.qid = GetManager().Add(queue, RedisUniqueQueueType, config, exemplar) - - return queue, nil -} - -var _ UniqueByteFIFO = &RedisUniqueByteFIFO{} - -// RedisUniqueByteFIFO represents a UniqueByteFIFO formed from a redisClient -type RedisUniqueByteFIFO struct { - RedisByteFIFO - setName string -} - -// RedisUniqueByteFIFOConfiguration is the configuration for the RedisUniqueByteFIFO -type RedisUniqueByteFIFOConfiguration struct { - RedisByteFIFOConfiguration - SetName string -} - -// NewRedisUniqueByteFIFO creates a UniqueByteFIFO formed from a redisClient -func NewRedisUniqueByteFIFO(config RedisUniqueByteFIFOConfiguration) (*RedisUniqueByteFIFO, error) { - internal, err := NewRedisByteFIFO(config.RedisByteFIFOConfiguration) - if err != nil { - return nil, err - } - - fifo := &RedisUniqueByteFIFO{ - RedisByteFIFO: *internal, - setName: config.SetName, - } - - return fifo, nil -} - -// PushFunc pushes data to the end of the fifo and calls the callback if it is added -func (fifo *RedisUniqueByteFIFO) PushFunc(ctx context.Context, data []byte, fn func() error) error { - added, err := fifo.client.SAdd(ctx, fifo.setName, data).Result() - if err != nil { - return err - } - if added == 0 { - return ErrAlreadyInQueue - } - if fn != nil { - if err := fn(); err != nil { - return err - } - } - return fifo.client.RPush(ctx, fifo.queueName, data).Err() -} - -// PushBack pushes data to the top of the fifo -func (fifo *RedisUniqueByteFIFO) PushBack(ctx context.Context, data []byte) error { - added, err := fifo.client.SAdd(ctx, fifo.setName, data).Result() - if err != nil { - return err - } - if added == 0 { - return ErrAlreadyInQueue - } - return fifo.client.LPush(ctx, fifo.queueName, data).Err() -} - -// Pop pops data from the start of the fifo -func (fifo *RedisUniqueByteFIFO) Pop(ctx context.Context) ([]byte, error) { - data, err := fifo.client.LPop(ctx, fifo.queueName).Bytes() - if err != nil && err != redis.Nil { - return data, err - } - - if len(data) == 0 { - return data, nil - } - - err = fifo.client.SRem(ctx, fifo.setName, data).Err() - return data, err -} - -// Has returns whether the fifo contains this data -func (fifo *RedisUniqueByteFIFO) Has(ctx context.Context, data []byte) (bool, error) { - return fifo.client.SIsMember(ctx, fifo.setName, data).Result() -} - -func init() { - queuesMap[RedisUniqueQueueType] = NewRedisUniqueQueue -} diff --git a/modules/queue/unique_queue_wrapped.go b/modules/queue/unique_queue_wrapped.go deleted file mode 100644 index 22eeb75c4..000000000 --- a/modules/queue/unique_queue_wrapped.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "fmt" - "sync" - "time" -) - -// WrappedUniqueQueueType is the type for a wrapped delayed starting queue -const WrappedUniqueQueueType Type = "unique-wrapped" - -// WrappedUniqueQueueConfiguration is the configuration for a WrappedUniqueQueue -type WrappedUniqueQueueConfiguration struct { - Underlying Type - Timeout time.Duration - MaxAttempts int - Config interface{} - QueueLength int - Name string -} - -// WrappedUniqueQueue wraps a delayed starting unique queue -type WrappedUniqueQueue struct { - *WrappedQueue - table map[Data]bool - tlock sync.Mutex - ready bool -} - -// NewWrappedUniqueQueue will attempt to create a unique queue of the provided type, -// but if there is a problem creating this queue it will instead create -// a WrappedUniqueQueue with delayed startup of the queue instead and a -// channel which will be redirected to the queue -// -// Please note that this Queue does not guarantee that a particular -// task cannot be processed twice or more at the same time. Uniqueness is -// only guaranteed whilst the task is waiting in the queue. -func NewWrappedUniqueQueue(handle HandlerFunc, cfg, exemplar interface{}) (Queue, error) { - configInterface, err := toConfig(WrappedUniqueQueueConfiguration{}, cfg) - if err != nil { - return nil, err - } - config := configInterface.(WrappedUniqueQueueConfiguration) - - queue, err := NewQueue(config.Underlying, handle, config.Config, exemplar) - if err == nil { - // Just return the queue there is no need to wrap - return queue, nil - } - if IsErrInvalidConfiguration(err) { - // Retrying ain't gonna make this any better... - return nil, ErrInvalidConfiguration{cfg: cfg} - } - - wrapped := &WrappedUniqueQueue{ - WrappedQueue: &WrappedQueue{ - channel: make(chan Data, config.QueueLength), - exemplar: exemplar, - delayedStarter: delayedStarter{ - cfg: config.Config, - underlying: config.Underlying, - timeout: config.Timeout, - maxAttempts: config.MaxAttempts, - name: config.Name, - }, - }, - table: map[Data]bool{}, - } - - // wrapped.handle is passed to the delayedStarting internal queue and is run to handle - // data passed to - wrapped.handle = func(data ...Data) (unhandled []Data) { - for _, datum := range data { - wrapped.tlock.Lock() - if !wrapped.ready { - delete(wrapped.table, data) - // If our table is empty all of the requests we have buffered between the - // wrapper queue starting and the internal queue starting have been handled. - // We can stop buffering requests in our local table and just pass Push - // direct to the internal queue - if len(wrapped.table) == 0 { - wrapped.ready = true - } - } - wrapped.tlock.Unlock() - if u := handle(datum); u != nil { - unhandled = append(unhandled, u...) - } - } - return unhandled - } - _ = GetManager().Add(queue, WrappedUniqueQueueType, config, exemplar) - return wrapped, nil -} - -// Push will push the data to the internal channel checking it against the exemplar -func (q *WrappedUniqueQueue) Push(data Data) error { - return q.PushFunc(data, nil) -} - -// PushFunc will push the data to the internal channel checking it against the exemplar -func (q *WrappedUniqueQueue) PushFunc(data Data, fn func() error) error { - if !assignableTo(data, q.exemplar) { - return fmt.Errorf("unable to assign data: %v to same type as exemplar: %v in %s", data, q.exemplar, q.name) - } - - q.tlock.Lock() - if q.ready { - // ready means our table is empty and all of the requests we have buffered between the - // wrapper queue starting and the internal queue starting have been handled. - // We can stop buffering requests in our local table and just pass Push - // direct to the internal queue - q.tlock.Unlock() - return q.internal.(UniqueQueue).PushFunc(data, fn) - } - - locked := true - defer func() { - if locked { - q.tlock.Unlock() - } - }() - if _, ok := q.table[data]; ok { - return ErrAlreadyInQueue - } - // FIXME: We probably need to implement some sort of limit here - // If the downstream queue blocks this table will grow without limit - q.table[data] = true - if fn != nil { - err := fn() - if err != nil { - delete(q.table, data) - return err - } - } - locked = false - q.tlock.Unlock() - - q.channel <- data - return nil -} - -// Has checks if the data is in the queue -func (q *WrappedUniqueQueue) Has(data Data) (bool, error) { - q.tlock.Lock() - defer q.tlock.Unlock() - if q.ready { - return q.internal.(UniqueQueue).Has(data) - } - _, has := q.table[data] - return has, nil -} - -// IsEmpty checks whether the queue is empty -func (q *WrappedUniqueQueue) IsEmpty() bool { - q.tlock.Lock() - if len(q.table) > 0 { - q.tlock.Unlock() - return false - } - if q.ready { - q.tlock.Unlock() - return q.internal.IsEmpty() - } - q.tlock.Unlock() - return false -} - -func init() { - queuesMap[WrappedUniqueQueueType] = NewWrappedUniqueQueue -} diff --git a/modules/queue/workergroup.go b/modules/queue/workergroup.go new file mode 100644 index 000000000..7127ea111 --- /dev/null +++ b/modules/queue/workergroup.go @@ -0,0 +1,331 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "code.gitea.io/gitea/modules/log" +) + +var ( + infiniteTimerC = make(chan time.Time) + batchDebounceDuration = 100 * time.Millisecond + workerIdleDuration = 1 * time.Second + + unhandledItemRequeueDuration atomic.Int64 // to avoid data race during test +) + +func init() { + unhandledItemRequeueDuration.Store(int64(5 * time.Second)) +} + +// workerGroup is a group of workers to work with a WorkerPoolQueue +type workerGroup[T any] struct { + q *WorkerPoolQueue[T] + wg sync.WaitGroup + + ctxWorker context.Context + ctxWorkerCancel context.CancelFunc + + batchBuffer []T + popItemChan chan []byte + popItemErr chan error +} + +func (wg *workerGroup[T]) doPrepareWorkerContext() { + wg.ctxWorker, wg.ctxWorkerCancel = context.WithCancel(wg.q.ctxRun) +} + +// doDispatchBatchToWorker dispatches a batch of items to worker's channel. +// If the channel is full, it tries to start a new worker if possible. +func (q *WorkerPoolQueue[T]) doDispatchBatchToWorker(wg *workerGroup[T], flushChan chan flushType) { + batch := wg.batchBuffer + wg.batchBuffer = nil + + if len(batch) == 0 { + return + } + + full := false + select { + case q.batchChan <- batch: + default: + full = true + } + + q.workerNumMu.Lock() + noWorker := q.workerNum == 0 + if full || noWorker { + if q.workerNum < q.workerMaxNum || noWorker && q.workerMaxNum <= 0 { + q.workerNum++ + q.doStartNewWorker(wg) + } + } + q.workerNumMu.Unlock() + + if full { + select { + case q.batchChan <- batch: + case flush := <-flushChan: + q.doWorkerHandle(batch) + q.doFlush(wg, flush) + case <-q.ctxRun.Done(): + wg.batchBuffer = batch // return the batch to buffer, the "doRun" function will handle it + } + } +} + +// doWorkerHandle calls the safeHandler to handle a batch of items, and it increases/decreases the active worker number. +// If the context has been canceled, it should not be caller because the "Push" still needs the context, in such case, call q.safeHandler directly +func (q *WorkerPoolQueue[T]) doWorkerHandle(batch []T) { + q.workerNumMu.Lock() + q.workerActiveNum++ + q.workerNumMu.Unlock() + + defer func() { + q.workerNumMu.Lock() + q.workerActiveNum-- + q.workerNumMu.Unlock() + }() + + unhandled := q.safeHandler(batch...) + // if none of the items were handled, it should back-off for a few seconds + // in this case the handler (eg: document indexer) may have encountered some errors/failures + if len(unhandled) == len(batch) && unhandledItemRequeueDuration.Load() != 0 { + log.Error("Queue %q failed to handle batch of %d items, backoff for a few seconds", q.GetName(), len(batch)) + select { + case <-q.ctxRun.Done(): + case <-time.After(time.Duration(unhandledItemRequeueDuration.Load())): + } + } + for _, item := range unhandled { + if err := q.Push(item); err != nil { + if !q.basePushForShutdown(item) { + log.Error("Failed to requeue item for queue %q when calling handler: %v", q.GetName(), err) + } + } + } +} + +// basePushForShutdown tries to requeue items into the base queue when the WorkerPoolQueue is shutting down. +// If the queue is shutting down, it returns true and try to push the items +// Otherwise it does nothing and returns false +func (q *WorkerPoolQueue[T]) basePushForShutdown(items ...T) bool { + ctxShutdown := q.ctxShutdown.Load() + if ctxShutdown == nil { + return false + } + for _, item := range items { + // if there is still any error, the queue can do nothing instead of losing the items + if err := q.baseQueue.PushItem(*ctxShutdown, q.marshal(item)); err != nil { + log.Error("Failed to requeue item for queue %q when shutting down: %v", q.GetName(), err) + } + } + return true +} + +// doStartNewWorker starts a new worker for the queue, the worker reads from worker's channel and handles the items. +func (q *WorkerPoolQueue[T]) doStartNewWorker(wp *workerGroup[T]) { + wp.wg.Add(1) + + go func() { + defer wp.wg.Done() + + log.Debug("Queue %q starts new worker", q.GetName()) + defer log.Debug("Queue %q stops idle worker", q.GetName()) + + t := time.NewTicker(workerIdleDuration) + keepWorking := true + stopWorking := func() { + q.workerNumMu.Lock() + keepWorking = false + q.workerNum-- + q.workerNumMu.Unlock() + } + for keepWorking { + select { + case <-wp.ctxWorker.Done(): + stopWorking() + case batch, ok := <-q.batchChan: + if !ok { + stopWorking() + } else { + q.doWorkerHandle(batch) + t.Reset(workerIdleDuration) + } + case <-t.C: + q.workerNumMu.Lock() + keepWorking = q.workerNum <= 1 + if !keepWorking { + q.workerNum-- + } + q.workerNumMu.Unlock() + } + } + }() +} + +// doFlush flushes the queue: it tries to read all items from the queue and handles them. +// It is for testing purpose only. It's not designed to work for a cluster. +func (q *WorkerPoolQueue[T]) doFlush(wg *workerGroup[T], flush flushType) { + log.Debug("Queue %q starts flushing", q.GetName()) + defer log.Debug("Queue %q finishes flushing", q.GetName()) + + // stop all workers, and prepare a new worker context to start new workers + + wg.ctxWorkerCancel() + wg.wg.Wait() + + defer func() { + close(flush) + wg.doPrepareWorkerContext() + }() + + // drain the batch channel first +loop: + for { + select { + case batch := <-q.batchChan: + q.doWorkerHandle(batch) + default: + break loop + } + } + + // drain the popItem channel + emptyCounter := 0 + for { + select { + case data, dataOk := <-wg.popItemChan: + if !dataOk { + return + } + emptyCounter = 0 + if v, jsonOk := q.unmarshal(data); !jsonOk { + continue + } else { + q.doWorkerHandle([]T{v}) + } + case err := <-wg.popItemErr: + if !q.isCtxRunCanceled() { + log.Error("Failed to pop item from queue %q (doFlush): %v", q.GetName(), err) + } + return + case <-q.ctxRun.Done(): + log.Debug("Queue %q is shutting down", q.GetName()) + return + case <-time.After(20 * time.Millisecond): + // There is no reliable way to make sure all queue items are consumed by the Flush, there always might be some items stored in some buffers/temp variables. + // If we run Gitea in a cluster, we can even not guarantee all items are consumed in a deterministic instance. + // Luckily, the "Flush" trick is only used in tests, so far so good. + if cnt, _ := q.baseQueue.Len(q.ctxRun); cnt == 0 && len(wg.popItemChan) == 0 { + emptyCounter++ + } + if emptyCounter >= 2 { + return + } + } + } +} + +func (q *WorkerPoolQueue[T]) isCtxRunCanceled() bool { + select { + case <-q.ctxRun.Done(): + return true + default: + return false + } +} + +var skipFlushChan = make(chan flushType) // an empty flush chan, used to skip reading other flush requests + +// doRun is the main loop of the queue. All related "doXxx" functions are executed in its context. +func (q *WorkerPoolQueue[T]) doRun() { + log.Debug("Queue %q starts running", q.GetName()) + defer log.Debug("Queue %q stops running", q.GetName()) + + wg := &workerGroup[T]{q: q} + wg.doPrepareWorkerContext() + wg.popItemChan, wg.popItemErr = popItemByChan(q.ctxRun, q.baseQueue.PopItem) + + defer func() { + q.ctxRunCancel() + + // drain all data on the fly + // since the queue is shutting down, the items can't be dispatched to workers because the context is canceled + // it can't call doWorkerHandle either, because there is no chance to push unhandled items back to the queue + var unhandled []T + close(q.batchChan) + for batch := range q.batchChan { + unhandled = append(unhandled, batch...) + } + unhandled = append(unhandled, wg.batchBuffer...) + for data := range wg.popItemChan { + if v, ok := q.unmarshal(data); ok { + unhandled = append(unhandled, v) + } + } + + ctxShutdownPtr := q.ctxShutdown.Load() + if ctxShutdownPtr != nil { + // if there is a shutdown context, try to push the items back to the base queue + q.basePushForShutdown(unhandled...) + workerDone := make(chan struct{}) + // the only way to wait for the workers, because the handlers do not have context to wait for + go func() { wg.wg.Wait(); close(workerDone) }() + select { + case <-workerDone: + case <-(*ctxShutdownPtr).Done(): + log.Error("Queue %q is shutting down, but workers are still running after timeout", q.GetName()) + } + } else { + // if there is no shutdown context, just call the handler to try to handle the items. if the handler fails again, the items are lost + q.safeHandler(unhandled...) + } + + close(q.shutdownDone) + }() + + var batchDispatchC <-chan time.Time = infiniteTimerC + for { + select { + case data, dataOk := <-wg.popItemChan: + if !dataOk { + return + } + if v, jsonOk := q.unmarshal(data); !jsonOk { + testRecorder.Record("pop:corrupted:%s", data) // in rare cases the levelqueue(leveldb) might be corrupted + continue + } else { + wg.batchBuffer = append(wg.batchBuffer, v) + } + if len(wg.batchBuffer) >= q.batchLength { + q.doDispatchBatchToWorker(wg, q.flushChan) + } else if batchDispatchC == infiniteTimerC { + batchDispatchC = time.After(batchDebounceDuration) + } // else: batchDispatchC is already a debounce timer, it will be triggered soon + case <-batchDispatchC: + batchDispatchC = infiniteTimerC + q.doDispatchBatchToWorker(wg, q.flushChan) + case flush := <-q.flushChan: + // before flushing, it needs to try to dispatch the batch to worker first, in case there is no worker running + // after the flushing, there is at least one worker running, so "doFlush" could wait for workers to finish + // since we are already in a "flush" operation, so the dispatching function shouldn't read the flush chan. + q.doDispatchBatchToWorker(wg, skipFlushChan) + q.doFlush(wg, flush) + case err := <-wg.popItemErr: + if !q.isCtxRunCanceled() { + log.Error("Failed to pop item from queue %q (doRun): %v", q.GetName(), err) + } + return + case <-q.ctxRun.Done(): + log.Debug("Queue %q is shutting down", q.GetName()) + return + } + } +} diff --git a/modules/queue/workerpool.go b/modules/queue/workerpool.go deleted file mode 100644 index b32128cb8..000000000 --- a/modules/queue/workerpool.go +++ /dev/null @@ -1,613 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package queue - -import ( - "context" - "fmt" - "runtime/pprof" - "sync" - "sync/atomic" - "time" - - "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/process" - "code.gitea.io/gitea/modules/util" -) - -// WorkerPool represent a dynamically growable worker pool for a -// provided handler function. They have an internal channel which -// they use to detect if there is a block and will grow and shrink in -// response to demand as per configuration. -type WorkerPool struct { - // This field requires to be the first one in the struct. - // This is to allow 64 bit atomic operations on 32-bit machines. - // See: https://pkg.go.dev/sync/atomic#pkg-note-BUG & Gitea issue 19518 - numInQueue int64 - lock sync.Mutex - baseCtx context.Context - baseCtxCancel context.CancelFunc - baseCtxFinished process.FinishedFunc - paused chan struct{} - resumed chan struct{} - cond *sync.Cond - qid int64 - maxNumberOfWorkers int - numberOfWorkers int - batchLength int - handle HandlerFunc - dataChan chan Data - blockTimeout time.Duration - boostTimeout time.Duration - boostWorkers int -} - -var ( - _ Flushable = &WorkerPool{} - _ ManagedPool = &WorkerPool{} -) - -// WorkerPoolConfiguration is the basic configuration for a WorkerPool -type WorkerPoolConfiguration struct { - Name string - QueueLength int - BatchLength int - BlockTimeout time.Duration - BoostTimeout time.Duration - BoostWorkers int - MaxWorkers int -} - -// NewWorkerPool creates a new worker pool -func NewWorkerPool(handle HandlerFunc, config WorkerPoolConfiguration) *WorkerPool { - ctx, cancel, finished := process.GetManager().AddTypedContext(context.Background(), fmt.Sprintf("Queue: %s", config.Name), process.SystemProcessType, false) - - dataChan := make(chan Data, config.QueueLength) - pool := &WorkerPool{ - baseCtx: ctx, - baseCtxCancel: cancel, - baseCtxFinished: finished, - batchLength: config.BatchLength, - dataChan: dataChan, - resumed: closedChan, - paused: make(chan struct{}), - handle: handle, - blockTimeout: config.BlockTimeout, - boostTimeout: config.BoostTimeout, - boostWorkers: config.BoostWorkers, - maxNumberOfWorkers: config.MaxWorkers, - } - - return pool -} - -// Done returns when this worker pool's base context has been cancelled -func (p *WorkerPool) Done() <-chan struct{} { - return p.baseCtx.Done() -} - -// Push pushes the data to the internal channel -func (p *WorkerPool) Push(data Data) { - atomic.AddInt64(&p.numInQueue, 1) - p.lock.Lock() - select { - case <-p.paused: - p.lock.Unlock() - p.dataChan <- data - return - default: - } - - if p.blockTimeout > 0 && p.boostTimeout > 0 && (p.numberOfWorkers <= p.maxNumberOfWorkers || p.maxNumberOfWorkers < 0) { - if p.numberOfWorkers == 0 { - p.zeroBoost() - } else { - p.lock.Unlock() - } - p.pushBoost(data) - } else { - p.lock.Unlock() - p.dataChan <- data - } -} - -// HasNoWorkerScaling will return true if the queue has no workers, and has no worker boosting -func (p *WorkerPool) HasNoWorkerScaling() bool { - p.lock.Lock() - defer p.lock.Unlock() - return p.hasNoWorkerScaling() -} - -func (p *WorkerPool) hasNoWorkerScaling() bool { - return p.numberOfWorkers == 0 && (p.boostTimeout == 0 || p.boostWorkers == 0 || p.maxNumberOfWorkers == 0) -} - -// zeroBoost will add a temporary boost worker for a no worker queue -// p.lock must be locked at the start of this function BUT it will be unlocked by the end of this function -// (This is because addWorkers has to be called whilst unlocked) -func (p *WorkerPool) zeroBoost() { - ctx, cancel := context.WithTimeout(p.baseCtx, p.boostTimeout) - mq := GetManager().GetManagedQueue(p.qid) - boost := p.boostWorkers - if (boost+p.numberOfWorkers) > p.maxNumberOfWorkers && p.maxNumberOfWorkers >= 0 { - boost = p.maxNumberOfWorkers - p.numberOfWorkers - } - if mq != nil { - log.Debug("WorkerPool: %d (for %s) has zero workers - adding %d temporary workers for %s", p.qid, mq.Name, boost, p.boostTimeout) - - start := time.Now() - pid := mq.RegisterWorkers(boost, start, true, start.Add(p.boostTimeout), cancel, false) - cancel = func() { - mq.RemoveWorkers(pid) - } - } else { - log.Debug("WorkerPool: %d has zero workers - adding %d temporary workers for %s", p.qid, p.boostWorkers, p.boostTimeout) - } - p.lock.Unlock() - p.addWorkers(ctx, cancel, boost) -} - -func (p *WorkerPool) pushBoost(data Data) { - select { - case p.dataChan <- data: - default: - p.lock.Lock() - if p.blockTimeout <= 0 { - p.lock.Unlock() - p.dataChan <- data - return - } - ourTimeout := p.blockTimeout - timer := time.NewTimer(p.blockTimeout) - p.lock.Unlock() - select { - case p.dataChan <- data: - util.StopTimer(timer) - case <-timer.C: - p.lock.Lock() - if p.blockTimeout > ourTimeout || (p.numberOfWorkers > p.maxNumberOfWorkers && p.maxNumberOfWorkers >= 0) { - p.lock.Unlock() - p.dataChan <- data - return - } - p.blockTimeout *= 2 - boostCtx, boostCtxCancel := context.WithCancel(p.baseCtx) - mq := GetManager().GetManagedQueue(p.qid) - boost := p.boostWorkers - if (boost+p.numberOfWorkers) > p.maxNumberOfWorkers && p.maxNumberOfWorkers >= 0 { - boost = p.maxNumberOfWorkers - p.numberOfWorkers - } - if mq != nil { - log.Debug("WorkerPool: %d (for %s) Channel blocked for %v - adding %d temporary workers for %s, block timeout now %v", p.qid, mq.Name, ourTimeout, boost, p.boostTimeout, p.blockTimeout) - - start := time.Now() - pid := mq.RegisterWorkers(boost, start, true, start.Add(p.boostTimeout), boostCtxCancel, false) - go func() { - <-boostCtx.Done() - mq.RemoveWorkers(pid) - boostCtxCancel() - }() - } else { - log.Debug("WorkerPool: %d Channel blocked for %v - adding %d temporary workers for %s, block timeout now %v", p.qid, ourTimeout, p.boostWorkers, p.boostTimeout, p.blockTimeout) - } - go func() { - <-time.After(p.boostTimeout) - boostCtxCancel() - p.lock.Lock() - p.blockTimeout /= 2 - p.lock.Unlock() - }() - p.lock.Unlock() - p.addWorkers(boostCtx, boostCtxCancel, boost) - p.dataChan <- data - } - } -} - -// NumberOfWorkers returns the number of current workers in the pool -func (p *WorkerPool) NumberOfWorkers() int { - p.lock.Lock() - defer p.lock.Unlock() - return p.numberOfWorkers -} - -// NumberInQueue returns the number of items in the queue -func (p *WorkerPool) NumberInQueue() int64 { - return atomic.LoadInt64(&p.numInQueue) -} - -// MaxNumberOfWorkers returns the maximum number of workers automatically added to the pool -func (p *WorkerPool) MaxNumberOfWorkers() int { - p.lock.Lock() - defer p.lock.Unlock() - return p.maxNumberOfWorkers -} - -// BoostWorkers returns the number of workers for a boost -func (p *WorkerPool) BoostWorkers() int { - p.lock.Lock() - defer p.lock.Unlock() - return p.boostWorkers -} - -// BoostTimeout returns the timeout of the next boost -func (p *WorkerPool) BoostTimeout() time.Duration { - p.lock.Lock() - defer p.lock.Unlock() - return p.boostTimeout -} - -// BlockTimeout returns the timeout til the next boost -func (p *WorkerPool) BlockTimeout() time.Duration { - p.lock.Lock() - defer p.lock.Unlock() - return p.blockTimeout -} - -// SetPoolSettings sets the setable boost values -func (p *WorkerPool) SetPoolSettings(maxNumberOfWorkers, boostWorkers int, timeout time.Duration) { - p.lock.Lock() - defer p.lock.Unlock() - p.maxNumberOfWorkers = maxNumberOfWorkers - p.boostWorkers = boostWorkers - p.boostTimeout = timeout -} - -// SetMaxNumberOfWorkers sets the maximum number of workers automatically added to the pool -// Changing this number will not change the number of current workers but will change the limit -// for future additions -func (p *WorkerPool) SetMaxNumberOfWorkers(newMax int) { - p.lock.Lock() - defer p.lock.Unlock() - p.maxNumberOfWorkers = newMax -} - -func (p *WorkerPool) commonRegisterWorkers(number int, timeout time.Duration, isFlusher bool) (context.Context, context.CancelFunc) { - var ctx context.Context - var cancel context.CancelFunc - start := time.Now() - end := start - hasTimeout := false - if timeout > 0 { - ctx, cancel = context.WithTimeout(p.baseCtx, timeout) - end = start.Add(timeout) - hasTimeout = true - } else { - ctx, cancel = context.WithCancel(p.baseCtx) - } - - mq := GetManager().GetManagedQueue(p.qid) - if mq != nil { - pid := mq.RegisterWorkers(number, start, hasTimeout, end, cancel, isFlusher) - log.Trace("WorkerPool: %d (for %s) adding %d workers with group id: %d", p.qid, mq.Name, number, pid) - return ctx, func() { - mq.RemoveWorkers(pid) - } - } - log.Trace("WorkerPool: %d adding %d workers (no group id)", p.qid, number) - - return ctx, cancel -} - -// AddWorkers adds workers to the pool - this allows the number of workers to go above the limit -func (p *WorkerPool) AddWorkers(number int, timeout time.Duration) context.CancelFunc { - ctx, cancel := p.commonRegisterWorkers(number, timeout, false) - p.addWorkers(ctx, cancel, number) - return cancel -} - -// addWorkers adds workers to the pool -func (p *WorkerPool) addWorkers(ctx context.Context, cancel context.CancelFunc, number int) { - for i := 0; i < number; i++ { - p.lock.Lock() - if p.cond == nil { - p.cond = sync.NewCond(&p.lock) - } - p.numberOfWorkers++ - p.lock.Unlock() - go func() { - pprof.SetGoroutineLabels(ctx) - p.doWork(ctx) - - p.lock.Lock() - p.numberOfWorkers-- - if p.numberOfWorkers == 0 { - p.cond.Broadcast() - cancel() - } else if p.numberOfWorkers < 0 { - // numberOfWorkers can't go negative but... - log.Warn("Number of Workers < 0 for QID %d - this shouldn't happen", p.qid) - p.numberOfWorkers = 0 - p.cond.Broadcast() - cancel() - } - select { - case <-p.baseCtx.Done(): - // Don't warn or check for ongoing work if the baseCtx is shutdown - case <-p.paused: - // Don't warn or check for ongoing work if the pool is paused - default: - if p.hasNoWorkerScaling() { - log.Warn( - "Queue: %d is configured to be non-scaling and has no workers - this configuration is likely incorrect.\n"+ - "The queue will be paused to prevent data-loss with the assumption that you will add workers and unpause as required.", p.qid) - p.pause() - } else if p.numberOfWorkers == 0 && atomic.LoadInt64(&p.numInQueue) > 0 { - // OK there are no workers but... there's still work to be done -> Reboost - p.zeroBoost() - // p.lock will be unlocked by zeroBoost - return - } - } - p.lock.Unlock() - }() - } -} - -// Wait for WorkerPool to finish -func (p *WorkerPool) Wait() { - p.lock.Lock() - defer p.lock.Unlock() - if p.cond == nil { - p.cond = sync.NewCond(&p.lock) - } - if p.numberOfWorkers <= 0 { - return - } - p.cond.Wait() -} - -// IsPaused returns if the pool is paused -func (p *WorkerPool) IsPaused() bool { - p.lock.Lock() - defer p.lock.Unlock() - select { - case <-p.paused: - return true - default: - return false - } -} - -// IsPausedIsResumed returns if the pool is paused and a channel that is closed when it is resumed -func (p *WorkerPool) IsPausedIsResumed() (<-chan struct{}, <-chan struct{}) { - p.lock.Lock() - defer p.lock.Unlock() - return p.paused, p.resumed -} - -// Pause pauses the WorkerPool -func (p *WorkerPool) Pause() { - p.lock.Lock() - defer p.lock.Unlock() - p.pause() -} - -func (p *WorkerPool) pause() { - select { - case <-p.paused: - default: - p.resumed = make(chan struct{}) - close(p.paused) - } -} - -// Resume resumes the WorkerPool -func (p *WorkerPool) Resume() { - p.lock.Lock() // can't defer unlock because of the zeroBoost at the end - select { - case <-p.resumed: - // already resumed - there's nothing to do - p.lock.Unlock() - return - default: - } - - p.paused = make(chan struct{}) - close(p.resumed) - - // OK now we need to check if we need to add some workers... - if p.numberOfWorkers > 0 || p.hasNoWorkerScaling() || atomic.LoadInt64(&p.numInQueue) == 0 { - // We either have workers, can't scale or there's no work to be done -> so just resume - p.lock.Unlock() - return - } - - // OK we got some work but no workers we need to think about boosting - select { - case <-p.baseCtx.Done(): - // don't bother boosting if the baseCtx is done - p.lock.Unlock() - return - default: - } - - // OK we'd better add some boost workers! - p.zeroBoost() - // p.zeroBoost will unlock the lock -} - -// CleanUp will drain the remaining contents of the channel -// This should be called after AddWorkers context is closed -func (p *WorkerPool) CleanUp(ctx context.Context) { - log.Trace("WorkerPool: %d CleanUp", p.qid) - close(p.dataChan) - for data := range p.dataChan { - if unhandled := p.handle(data); unhandled != nil { - if unhandled != nil { - log.Error("Unhandled Data in clean-up of queue %d", p.qid) - } - } - - atomic.AddInt64(&p.numInQueue, -1) - select { - case <-ctx.Done(): - log.Warn("WorkerPool: %d Cleanup context closed before finishing clean-up", p.qid) - return - default: - } - } - log.Trace("WorkerPool: %d CleanUp Done", p.qid) -} - -// Flush flushes the channel with a timeout - the Flush worker will be registered as a flush worker with the manager -func (p *WorkerPool) Flush(timeout time.Duration) error { - ctx, cancel := p.commonRegisterWorkers(1, timeout, true) - defer cancel() - return p.FlushWithContext(ctx) -} - -// IsEmpty returns if true if the worker queue is empty -func (p *WorkerPool) IsEmpty() bool { - return atomic.LoadInt64(&p.numInQueue) == 0 -} - -// contextError returns either ctx.Done(), the base context's error or nil -func (p *WorkerPool) contextError(ctx context.Context) error { - select { - case <-p.baseCtx.Done(): - return p.baseCtx.Err() - case <-ctx.Done(): - return ctx.Err() - default: - return nil - } -} - -// FlushWithContext is very similar to CleanUp but it will return as soon as the dataChan is empty -// NB: The worker will not be registered with the manager. -func (p *WorkerPool) FlushWithContext(ctx context.Context) error { - log.Trace("WorkerPool: %d Flush", p.qid) - paused, _ := p.IsPausedIsResumed() - for { - // Because select will return any case that is satisified at random we precheck here before looking at dataChan. - select { - case <-paused: - // Ensure that even if paused that the cancelled error is still sent - return p.contextError(ctx) - case <-p.baseCtx.Done(): - return p.baseCtx.Err() - case <-ctx.Done(): - return ctx.Err() - default: - } - - select { - case <-paused: - return p.contextError(ctx) - case data, ok := <-p.dataChan: - if !ok { - return nil - } - if unhandled := p.handle(data); unhandled != nil { - log.Error("Unhandled Data whilst flushing queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1) - case <-p.baseCtx.Done(): - return p.baseCtx.Err() - case <-ctx.Done(): - return ctx.Err() - default: - return nil - } - } -} - -func (p *WorkerPool) doWork(ctx context.Context) { - pprof.SetGoroutineLabels(ctx) - delay := time.Millisecond * 300 - - // Create a common timer - we will use this elsewhere - timer := time.NewTimer(0) - util.StopTimer(timer) - - paused, _ := p.IsPausedIsResumed() - data := make([]Data, 0, p.batchLength) - for { - // Because select will return any case that is satisified at random we precheck here before looking at dataChan. - select { - case <-paused: - log.Trace("Worker for Queue %d Pausing", p.qid) - if len(data) > 0 { - log.Trace("Handling: %d data, %v", len(data), data) - if unhandled := p.handle(data...); unhandled != nil { - log.Error("Unhandled Data in queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) - } - _, resumed := p.IsPausedIsResumed() - select { - case <-resumed: - paused, _ = p.IsPausedIsResumed() - log.Trace("Worker for Queue %d Resuming", p.qid) - util.StopTimer(timer) - case <-ctx.Done(): - log.Trace("Worker shutting down") - return - } - case <-ctx.Done(): - if len(data) > 0 { - log.Trace("Handling: %d data, %v", len(data), data) - if unhandled := p.handle(data...); unhandled != nil { - log.Error("Unhandled Data in queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) - } - log.Trace("Worker shutting down") - return - default: - } - - select { - case <-paused: - // go back around - case <-ctx.Done(): - if len(data) > 0 { - log.Trace("Handling: %d data, %v", len(data), data) - if unhandled := p.handle(data...); unhandled != nil { - log.Error("Unhandled Data in queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) - } - log.Trace("Worker shutting down") - return - case datum, ok := <-p.dataChan: - if !ok { - // the dataChan has been closed - we should finish up: - if len(data) > 0 { - log.Trace("Handling: %d data, %v", len(data), data) - if unhandled := p.handle(data...); unhandled != nil { - log.Error("Unhandled Data in queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) - } - log.Trace("Worker shutting down") - return - } - data = append(data, datum) - util.StopTimer(timer) - - if len(data) >= p.batchLength { - log.Trace("Handling: %d data, %v", len(data), data) - if unhandled := p.handle(data...); unhandled != nil { - log.Error("Unhandled Data in queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) - data = make([]Data, 0, p.batchLength) - } else { - timer.Reset(delay) - } - case <-timer.C: - delay = time.Millisecond * 100 - if len(data) > 0 { - log.Trace("Handling: %d data, %v", len(data), data) - if unhandled := p.handle(data...); unhandled != nil { - log.Error("Unhandled Data in queue %d", p.qid) - } - atomic.AddInt64(&p.numInQueue, -1*int64(len(data))) - data = make([]Data, 0, p.batchLength) - } - } - } -} diff --git a/modules/queue/workerqueue.go b/modules/queue/workerqueue.go new file mode 100644 index 000000000..493bea17a --- /dev/null +++ b/modules/queue/workerqueue.go @@ -0,0 +1,241 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "time" + + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/json" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" +) + +// WorkerPoolQueue is a queue that uses a pool of workers to process items +// It can use different underlying (base) queue types +type WorkerPoolQueue[T any] struct { + ctxRun context.Context + ctxRunCancel context.CancelFunc + ctxShutdown atomic.Pointer[context.Context] + shutdownDone chan struct{} + + origHandler HandlerFuncT[T] + safeHandler HandlerFuncT[T] + + baseQueueType string + baseConfig *BaseConfig + baseQueue baseQueue + + batchChan chan []T + flushChan chan flushType + + batchLength int + workerNum int + workerMaxNum int + workerActiveNum int + workerNumMu sync.Mutex +} + +type flushType chan struct{} + +var _ ManagedWorkerPoolQueue = (*WorkerPoolQueue[any])(nil) + +func (q *WorkerPoolQueue[T]) GetName() string { + return q.baseConfig.ManagedName +} + +func (q *WorkerPoolQueue[T]) GetType() string { + return q.baseQueueType +} + +func (q *WorkerPoolQueue[T]) GetItemTypeName() string { + var t T + return fmt.Sprintf("%T", t) +} + +func (q *WorkerPoolQueue[T]) GetWorkerNumber() int { + q.workerNumMu.Lock() + defer q.workerNumMu.Unlock() + return q.workerNum +} + +func (q *WorkerPoolQueue[T]) GetWorkerActiveNumber() int { + q.workerNumMu.Lock() + defer q.workerNumMu.Unlock() + return q.workerActiveNum +} + +func (q *WorkerPoolQueue[T]) GetWorkerMaxNumber() int { + q.workerNumMu.Lock() + defer q.workerNumMu.Unlock() + return q.workerMaxNum +} + +func (q *WorkerPoolQueue[T]) SetWorkerMaxNumber(num int) { + q.workerNumMu.Lock() + defer q.workerNumMu.Unlock() + q.workerMaxNum = num +} + +func (q *WorkerPoolQueue[T]) GetQueueItemNumber() int { + cnt, err := q.baseQueue.Len(q.ctxRun) + if err != nil { + log.Error("Failed to get number of items in queue %q: %v", q.GetName(), err) + } + return cnt +} + +func (q *WorkerPoolQueue[T]) FlushWithContext(ctx context.Context, timeout time.Duration) (err error) { + if q.isBaseQueueDummy() { + return + } + + log.Debug("Try to flush queue %q with timeout %v", q.GetName(), timeout) + defer log.Debug("Finish flushing queue %q, err: %v", q.GetName(), err) + + var after <-chan time.Time + after = infiniteTimerC + if timeout > 0 { + after = time.After(timeout) + } + c := make(flushType) + + // send flush request + // if it blocks, it means that there is a flush in progress or the queue hasn't been started yet + select { + case q.flushChan <- c: + case <-ctx.Done(): + return ctx.Err() + case <-q.ctxRun.Done(): + return q.ctxRun.Err() + case <-after: + return context.DeadlineExceeded + } + + // wait for flush to finish + select { + case <-c: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-q.ctxRun.Done(): + return q.ctxRun.Err() + case <-after: + return context.DeadlineExceeded + } +} + +func (q *WorkerPoolQueue[T]) marshal(data T) []byte { + bs, err := json.Marshal(data) + if err != nil { + log.Error("Failed to marshal item for queue %q: %v", q.GetName(), err) + return nil + } + return bs +} + +func (q *WorkerPoolQueue[T]) unmarshal(data []byte) (t T, ok bool) { + if err := json.Unmarshal(data, &t); err != nil { + log.Error("Failed to unmarshal item from queue %q: %v", q.GetName(), err) + return t, false + } + return t, true +} + +func (q *WorkerPoolQueue[T]) isBaseQueueDummy() bool { + _, isDummy := q.baseQueue.(*baseDummy) + return isDummy +} + +// Push adds an item to the queue, it may block for a while and then returns an error if the queue is full +func (q *WorkerPoolQueue[T]) Push(data T) error { + if q.isBaseQueueDummy() && q.safeHandler != nil { + // FIXME: the "immediate" queue is only for testing, but it really causes problems because its behavior is different from a real queue. + // Even if tests pass, it doesn't mean that there is no bug in code. + if data, ok := q.unmarshal(q.marshal(data)); ok { + q.safeHandler(data) + } + } + return q.baseQueue.PushItem(q.ctxRun, q.marshal(data)) +} + +// Has only works for unique queues. Keep in mind that this check may not be reliable (due to lacking of proper transaction support) +// There could be a small chance that duplicate items appear in the queue +func (q *WorkerPoolQueue[T]) Has(data T) (bool, error) { + return q.baseQueue.HasItem(q.ctxRun, q.marshal(data)) +} + +func (q *WorkerPoolQueue[T]) Run(atShutdown, atTerminate func(func())) { + atShutdown(func() { + // in case some queue handlers are slow or have hanging bugs, at most wait for a short time + q.ShutdownWait(1 * time.Second) + }) + q.doRun() +} + +// ShutdownWait shuts down the queue, waits for all workers to finish their jobs, and pushes the unhandled items back to the base queue +// It waits for all workers (handlers) to finish their jobs, in case some buggy handlers would hang forever, a reasonable timeout is needed +func (q *WorkerPoolQueue[T]) ShutdownWait(timeout time.Duration) { + shutdownCtx, shutdownCtxCancel := context.WithTimeout(context.Background(), timeout) + defer shutdownCtxCancel() + if q.ctxShutdown.CompareAndSwap(nil, &shutdownCtx) { + q.ctxRunCancel() + } + <-q.shutdownDone +} + +func getNewQueueFn(t string) (string, func(cfg *BaseConfig, unique bool) (baseQueue, error)) { + switch t { + case "dummy", "immediate": + return t, newBaseDummy + case "channel": + return t, newBaseChannelGeneric + case "redis": + return t, newBaseRedisGeneric + default: // level(leveldb,levelqueue,persistable-channel) + return "level", newBaseLevelQueueGeneric + } +} + +func NewWorkerPoolQueueBySetting[T any](name string, queueSetting setting.QueueSettings, handler HandlerFuncT[T], unique bool) (*WorkerPoolQueue[T], error) { + if handler == nil { + log.Debug("Use dummy queue for %q because handler is nil and caller doesn't want to process the queue items", name) + queueSetting.Type = "dummy" + } + + var w WorkerPoolQueue[T] + var err error + queueType, newQueueFn := getNewQueueFn(queueSetting.Type) + w.baseQueueType = queueType + w.baseConfig = toBaseConfig(name, queueSetting) + w.baseQueue, err = newQueueFn(w.baseConfig, unique) + if err != nil { + return nil, err + } + log.Trace("Created queue %q of type %q", name, queueType) + + w.ctxRun, w.ctxRunCancel = context.WithCancel(graceful.GetManager().ShutdownContext()) + w.batchChan = make(chan []T) + w.flushChan = make(chan flushType) + w.shutdownDone = make(chan struct{}) + w.workerMaxNum = queueSetting.MaxWorkers + w.batchLength = queueSetting.BatchLength + + w.origHandler = handler + w.safeHandler = func(t ...T) (unhandled []T) { + defer func() { + err := recover() + if err != nil { + log.Error("Recovered from panic in queue %q handler: %v\n%s", name, err, log.Stack(2)) + } + }() + return w.origHandler(t...) + } + + return &w, nil +} diff --git a/modules/queue/workerqueue_test.go b/modules/queue/workerqueue_test.go new file mode 100644 index 000000000..da9451cd7 --- /dev/null +++ b/modules/queue/workerqueue_test.go @@ -0,0 +1,260 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package queue + +import ( + "context" + "strconv" + "sync" + "testing" + "time" + + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/assert" +) + +func runWorkerPoolQueue[T any](q *WorkerPoolQueue[T]) func() { + var stop func() + started := make(chan struct{}) + stopped := make(chan struct{}) + go func() { + q.Run(func(f func()) { stop = f; close(started) }, nil) + close(stopped) + }() + <-started + return func() { + stop() + <-stopped + } +} + +func TestWorkerPoolQueueUnhandled(t *testing.T) { + oldUnhandledItemRequeueDuration := unhandledItemRequeueDuration.Load() + unhandledItemRequeueDuration.Store(0) + defer unhandledItemRequeueDuration.Store(oldUnhandledItemRequeueDuration) + + mu := sync.Mutex{} + + test := func(t *testing.T, queueSetting setting.QueueSettings) { + queueSetting.Length = 100 + queueSetting.Type = "channel" + queueSetting.Datadir = t.TempDir() + "/test-queue" + m := map[int]int{} + + // odds are handled once, evens are handled twice + handler := func(items ...int) (unhandled []int) { + testRecorder.Record("handle:%v", items) + for _, item := range items { + mu.Lock() + if item%2 == 0 && m[item] == 0 { + unhandled = append(unhandled, item) + } + m[item]++ + mu.Unlock() + } + return unhandled + } + + q, _ := NewWorkerPoolQueueBySetting("test-workpoolqueue", queueSetting, handler, false) + stop := runWorkerPoolQueue(q) + for i := 0; i < queueSetting.Length; i++ { + testRecorder.Record("push:%v", i) + assert.NoError(t, q.Push(i)) + } + assert.NoError(t, q.FlushWithContext(context.Background(), 0)) + stop() + + ok := true + for i := 0; i < queueSetting.Length; i++ { + if i%2 == 0 { + ok = ok && assert.EqualValues(t, 2, m[i], "test %s: item %d", t.Name(), i) + } else { + ok = ok && assert.EqualValues(t, 1, m[i], "test %s: item %d", t.Name(), i) + } + } + if !ok { + t.Logf("m: %v", m) + t.Logf("records: %v", testRecorder.Records()) + } + testRecorder.Reset() + } + + runCount := 2 // we can run these tests even hundreds times to see its stability + t.Run("1/1", func(t *testing.T) { + for i := 0; i < runCount; i++ { + test(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1}) + } + }) + t.Run("3/1", func(t *testing.T) { + for i := 0; i < runCount; i++ { + test(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1}) + } + }) + t.Run("4/5", func(t *testing.T) { + for i := 0; i < runCount; i++ { + test(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5}) + } + }) +} + +func TestWorkerPoolQueuePersistence(t *testing.T) { + runCount := 2 // we can run these tests even hundreds times to see its stability + t.Run("1/1", func(t *testing.T) { + for i := 0; i < runCount; i++ { + testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 1, MaxWorkers: 1, Length: 100}) + } + }) + t.Run("3/1", func(t *testing.T) { + for i := 0; i < runCount; i++ { + testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 3, MaxWorkers: 1, Length: 100}) + } + }) + t.Run("4/5", func(t *testing.T) { + for i := 0; i < runCount; i++ { + testWorkerPoolQueuePersistence(t, setting.QueueSettings{BatchLength: 4, MaxWorkers: 5, Length: 100}) + } + }) +} + +func testWorkerPoolQueuePersistence(t *testing.T, queueSetting setting.QueueSettings) { + testCount := queueSetting.Length + queueSetting.Type = "level" + queueSetting.Datadir = t.TempDir() + "/test-queue" + + mu := sync.Mutex{} + + var tasksQ1, tasksQ2 []string + q1 := func() { + startWhenAllReady := make(chan struct{}) // only start data consuming when the "testCount" tasks are all pushed into queue + stopAt20Shutdown := make(chan struct{}) // stop and shutdown at the 20th item + + testHandler := func(data ...string) []string { + <-startWhenAllReady + time.Sleep(10 * time.Millisecond) + for _, s := range data { + mu.Lock() + tasksQ1 = append(tasksQ1, s) + mu.Unlock() + + if s == "task-20" { + close(stopAt20Shutdown) + } + } + return nil + } + + q, _ := NewWorkerPoolQueueBySetting("pr_patch_checker_test", queueSetting, testHandler, true) + stop := runWorkerPoolQueue(q) + for i := 0; i < testCount; i++ { + _ = q.Push("task-" + strconv.Itoa(i)) + } + close(startWhenAllReady) + <-stopAt20Shutdown // it's possible to have more than 20 tasks executed + stop() + } + + q1() // run some tasks and shutdown at an intermediate point + + time.Sleep(100 * time.Millisecond) // because the handler in q1 has a slight delay, we need to wait for it to finish + + q2 := func() { + testHandler := func(data ...string) []string { + for _, s := range data { + mu.Lock() + tasksQ2 = append(tasksQ2, s) + mu.Unlock() + } + return nil + } + + q, _ := NewWorkerPoolQueueBySetting("pr_patch_checker_test", queueSetting, testHandler, true) + stop := runWorkerPoolQueue(q) + assert.NoError(t, q.FlushWithContext(context.Background(), 0)) + stop() + } + + q2() // restart the queue to continue to execute the tasks in it + + assert.NotZero(t, len(tasksQ1)) + assert.NotZero(t, len(tasksQ2)) + assert.EqualValues(t, testCount, len(tasksQ1)+len(tasksQ2)) +} + +func TestWorkerPoolQueueActiveWorkers(t *testing.T) { + oldWorkerIdleDuration := workerIdleDuration + workerIdleDuration = 300 * time.Millisecond + defer func() { + workerIdleDuration = oldWorkerIdleDuration + }() + + handler := func(items ...int) (unhandled []int) { + time.Sleep(100 * time.Millisecond) + return nil + } + + q, _ := NewWorkerPoolQueueBySetting("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 1, Length: 100}, handler, false) + stop := runWorkerPoolQueue(q) + for i := 0; i < 5; i++ { + assert.NoError(t, q.Push(i)) + } + + time.Sleep(50 * time.Millisecond) + assert.EqualValues(t, 1, q.GetWorkerNumber()) + assert.EqualValues(t, 1, q.GetWorkerActiveNumber()) + time.Sleep(500 * time.Millisecond) + assert.EqualValues(t, 1, q.GetWorkerNumber()) + assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + time.Sleep(workerIdleDuration) + assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working + stop() + + q, _ = NewWorkerPoolQueueBySetting("test-workpoolqueue", setting.QueueSettings{Type: "channel", BatchLength: 1, MaxWorkers: 3, Length: 100}, handler, false) + stop = runWorkerPoolQueue(q) + for i := 0; i < 15; i++ { + assert.NoError(t, q.Push(i)) + } + + time.Sleep(50 * time.Millisecond) + assert.EqualValues(t, 3, q.GetWorkerNumber()) + assert.EqualValues(t, 3, q.GetWorkerActiveNumber()) + time.Sleep(500 * time.Millisecond) + assert.EqualValues(t, 3, q.GetWorkerNumber()) + assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + time.Sleep(workerIdleDuration) + assert.EqualValues(t, 1, q.GetWorkerNumber()) // there is at least one worker after the queue begins working + stop() +} + +func TestWorkerPoolQueueShutdown(t *testing.T) { + oldUnhandledItemRequeueDuration := unhandledItemRequeueDuration.Load() + unhandledItemRequeueDuration.Store(int64(100 * time.Millisecond)) + defer unhandledItemRequeueDuration.Store(oldUnhandledItemRequeueDuration) + + // simulate a slow handler, it doesn't handle any item (all items will be pushed back to the queue) + handlerCalled := make(chan struct{}) + handler := func(items ...int) (unhandled []int) { + if items[0] == 0 { + close(handlerCalled) + } + time.Sleep(100 * time.Millisecond) + return items + } + + qs := setting.QueueSettings{Type: "level", Datadir: t.TempDir() + "/queue", BatchLength: 3, MaxWorkers: 4, Length: 20} + q, _ := NewWorkerPoolQueueBySetting("test-workpoolqueue", qs, handler, false) + stop := runWorkerPoolQueue(q) + for i := 0; i < qs.Length; i++ { + assert.NoError(t, q.Push(i)) + } + <-handlerCalled + time.Sleep(50 * time.Millisecond) // wait for a while to make sure all workers are active + assert.EqualValues(t, 4, q.GetWorkerActiveNumber()) + stop() // stop triggers shutdown + assert.EqualValues(t, 0, q.GetWorkerActiveNumber()) + + // no item was ever handled, so we still get all of them again + q, _ = NewWorkerPoolQueueBySetting("test-workpoolqueue", qs, handler, false) + assert.EqualValues(t, 20, q.GetQueueItemNumber()) +} diff --git a/modules/setting/config_provider.go b/modules/setting/config_provider.go index 168595829..24825a620 100644 --- a/modules/setting/config_provider.go +++ b/modules/setting/config_provider.go @@ -42,12 +42,12 @@ type iniFileConfigProvider struct { // NewEmptyConfigProvider create a new empty config provider func NewEmptyConfigProvider() ConfigProvider { - cp, _ := newConfigProviderFromData("") + cp, _ := NewConfigProviderFromData("") return cp } -// newConfigProviderFromData this function is only for testing -func newConfigProviderFromData(configContent string) (ConfigProvider, error) { +// NewConfigProviderFromData this function is only for testing +func NewConfigProviderFromData(configContent string) (ConfigProvider, error) { var cfg *ini.File var err error if configContent == "" { diff --git a/modules/setting/cron_test.go b/modules/setting/cron_test.go index 8d58cf8b4..3187ab18a 100644 --- a/modules/setting/cron_test.go +++ b/modules/setting/cron_test.go @@ -26,7 +26,7 @@ BASE = true SECOND = white rabbit EXTEND = true ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) extended := &Extended{ diff --git a/modules/setting/indexer.go b/modules/setting/indexer.go index 8aee8596d..6836e6231 100644 --- a/modules/setting/indexer.go +++ b/modules/setting/indexer.go @@ -70,15 +70,6 @@ func loadIndexerFrom(rootCfg ConfigProvider) { Indexer.IssueIndexerName = sec.Key("ISSUE_INDEXER_NAME").MustString(Indexer.IssueIndexerName) - // The following settings are deprecated and can be overridden by settings in [queue] or [queue.issue_indexer] - // DEPRECATED should not be removed because users maybe upgrade from lower version to the latest version - // if these are removed, the warning will not be shown - deprecatedSetting(rootCfg, "indexer", "ISSUE_INDEXER_QUEUE_TYPE", "queue.issue_indexer", "TYPE", "v1.19.0") - deprecatedSetting(rootCfg, "indexer", "ISSUE_INDEXER_QUEUE_DIR", "queue.issue_indexer", "DATADIR", "v1.19.0") - deprecatedSetting(rootCfg, "indexer", "ISSUE_INDEXER_QUEUE_CONN_STR", "queue.issue_indexer", "CONN_STR", "v1.19.0") - deprecatedSetting(rootCfg, "indexer", "ISSUE_INDEXER_QUEUE_BATCH_NUMBER", "queue.issue_indexer", "BATCH_LENGTH", "v1.19.0") - deprecatedSetting(rootCfg, "indexer", "UPDATE_BUFFER_LEN", "queue.issue_indexer", "LENGTH", "v1.19.0") - Indexer.RepoIndexerEnabled = sec.Key("REPO_INDEXER_ENABLED").MustBool(false) Indexer.RepoType = sec.Key("REPO_INDEXER_TYPE").MustString("bleve") Indexer.RepoPath = filepath.ToSlash(sec.Key("REPO_INDEXER_PATH").MustString(filepath.ToSlash(filepath.Join(AppDataPath, "indexers/repos.bleve")))) diff --git a/modules/setting/queue.go b/modules/setting/queue.go index 8c37e538b..8673537b5 100644 --- a/modules/setting/queue.go +++ b/modules/setting/queue.go @@ -5,198 +5,109 @@ package setting import ( "path/filepath" - "strconv" - "time" - "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" ) // QueueSettings represent the settings for a queue from the ini type QueueSettings struct { - Name string - DataDir string - QueueLength int `ini:"LENGTH"` - BatchLength int - ConnectionString string - Type string - QueueName string - SetName string - WrapIfNecessary bool - MaxAttempts int - Timeout time.Duration - Workers int - MaxWorkers int - BlockTimeout time.Duration - BoostTimeout time.Duration - BoostWorkers int + Name string // not an INI option, it is the name for [queue.the-name] section + + Type string + Datadir string + ConnStr string // for leveldb or redis + Length int // max queue length before blocking + + QueueName, SetName string // the name suffix for storage (db key, redis key), "set" is for unique queue + + BatchLength int + MaxWorkers int } -// Queue settings -var Queue = QueueSettings{} +var queueSettingsDefault = QueueSettings{ + Type: "level", // dummy, channel, level, redis + Datadir: "queues/common", // relative to AppDataPath + Length: 100, // queue length before a channel queue will block -// GetQueueSettings returns the queue settings for the appropriately named queue -func GetQueueSettings(name string) QueueSettings { - return getQueueSettings(CfgProvider, name) + QueueName: "_queue", + SetName: "_unique", + BatchLength: 20, + MaxWorkers: 10, } -func getQueueSettings(rootCfg ConfigProvider, name string) QueueSettings { - q := QueueSettings{} - sec := rootCfg.Section("queue." + name) - q.Name = name +func GetQueueSettings(rootCfg ConfigProvider, name string) (QueueSettings, error) { + // deep copy default settings + cfg := QueueSettings{} + if cfgBs, err := json.Marshal(queueSettingsDefault); err != nil { + return cfg, err + } else if err = json.Unmarshal(cfgBs, &cfg); err != nil { + return cfg, err + } - // DataDir is not directly inheritable - q.DataDir = filepath.ToSlash(filepath.Join(Queue.DataDir, "common")) - // QueueName is not directly inheritable either - q.QueueName = name + Queue.QueueName - for _, key := range sec.Keys() { - switch key.Name() { - case "DATADIR": - q.DataDir = key.MustString(q.DataDir) - case "QUEUE_NAME": - q.QueueName = key.MustString(q.QueueName) - case "SET_NAME": - q.SetName = key.MustString(q.SetName) + cfg.Name = name + if sec, err := rootCfg.GetSection("queue"); err == nil { + if err = sec.MapTo(&cfg); err != nil { + log.Error("Failed to map queue common config for %q: %v", name, err) + return cfg, nil } } - if len(q.SetName) == 0 && len(Queue.SetName) > 0 { - q.SetName = q.QueueName + Queue.SetName + if sec, err := rootCfg.GetSection("queue." + name); err == nil { + if err = sec.MapTo(&cfg); err != nil { + log.Error("Failed to map queue spec config for %q: %v", name, err) + return cfg, nil + } + if sec.HasKey("CONN_STR") { + cfg.ConnStr = sec.Key("CONN_STR").String() + } } - if !filepath.IsAbs(q.DataDir) { - q.DataDir = filepath.ToSlash(filepath.Join(AppDataPath, q.DataDir)) + + if cfg.Datadir == "" { + cfg.Datadir = queueSettingsDefault.Datadir } - _, _ = sec.NewKey("DATADIR", q.DataDir) + if !filepath.IsAbs(cfg.Datadir) { + cfg.Datadir = filepath.Join(AppDataPath, cfg.Datadir) + } + cfg.Datadir = filepath.ToSlash(cfg.Datadir) - // The rest are... - q.QueueLength = sec.Key("LENGTH").MustInt(Queue.QueueLength) - q.BatchLength = sec.Key("BATCH_LENGTH").MustInt(Queue.BatchLength) - q.ConnectionString = sec.Key("CONN_STR").MustString(Queue.ConnectionString) - q.Type = sec.Key("TYPE").MustString(Queue.Type) - q.WrapIfNecessary = sec.Key("WRAP_IF_NECESSARY").MustBool(Queue.WrapIfNecessary) - q.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Queue.MaxAttempts) - q.Timeout = sec.Key("TIMEOUT").MustDuration(Queue.Timeout) - q.Workers = sec.Key("WORKERS").MustInt(Queue.Workers) - q.MaxWorkers = sec.Key("MAX_WORKERS").MustInt(Queue.MaxWorkers) - q.BlockTimeout = sec.Key("BLOCK_TIMEOUT").MustDuration(Queue.BlockTimeout) - q.BoostTimeout = sec.Key("BOOST_TIMEOUT").MustDuration(Queue.BoostTimeout) - q.BoostWorkers = sec.Key("BOOST_WORKERS").MustInt(Queue.BoostWorkers) + if cfg.Type == "redis" && cfg.ConnStr == "" { + cfg.ConnStr = "redis://127.0.0.1:6379/0" + } - return q + if cfg.Length <= 0 { + cfg.Length = queueSettingsDefault.Length + } + if cfg.MaxWorkers <= 0 { + cfg.MaxWorkers = queueSettingsDefault.MaxWorkers + } + if cfg.BatchLength <= 0 { + cfg.BatchLength = queueSettingsDefault.BatchLength + } + + return cfg, nil } -// LoadQueueSettings sets up the default settings for Queues -// This is exported for tests to be able to use the queue func LoadQueueSettings() { loadQueueFrom(CfgProvider) } func loadQueueFrom(rootCfg ConfigProvider) { - sec := rootCfg.Section("queue") - Queue.DataDir = filepath.ToSlash(sec.Key("DATADIR").MustString("queues/")) - if !filepath.IsAbs(Queue.DataDir) { - Queue.DataDir = filepath.ToSlash(filepath.Join(AppDataPath, Queue.DataDir)) - } - Queue.QueueLength = sec.Key("LENGTH").MustInt(20) - Queue.BatchLength = sec.Key("BATCH_LENGTH").MustInt(20) - Queue.ConnectionString = sec.Key("CONN_STR").MustString("") - defaultType := sec.Key("TYPE").String() - Queue.Type = sec.Key("TYPE").MustString("persistable-channel") - Queue.WrapIfNecessary = sec.Key("WRAP_IF_NECESSARY").MustBool(true) - Queue.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(10) - Queue.Timeout = sec.Key("TIMEOUT").MustDuration(GracefulHammerTime + 30*time.Second) - Queue.Workers = sec.Key("WORKERS").MustInt(0) - Queue.MaxWorkers = sec.Key("MAX_WORKERS").MustInt(10) - Queue.BlockTimeout = sec.Key("BLOCK_TIMEOUT").MustDuration(1 * time.Second) - Queue.BoostTimeout = sec.Key("BOOST_TIMEOUT").MustDuration(5 * time.Minute) - Queue.BoostWorkers = sec.Key("BOOST_WORKERS").MustInt(1) - Queue.QueueName = sec.Key("QUEUE_NAME").MustString("_queue") - Queue.SetName = sec.Key("SET_NAME").MustString("") - - // Now handle the old issue_indexer configuration - // FIXME: DEPRECATED to be removed in v1.18.0 - section := rootCfg.Section("queue.issue_indexer") - directlySet := toDirectlySetKeysSet(section) - if !directlySet.Contains("TYPE") && defaultType == "" { - switch typ := rootCfg.Section("indexer").Key("ISSUE_INDEXER_QUEUE_TYPE").MustString(""); typ { - case "levelqueue": - _, _ = section.NewKey("TYPE", "level") - case "channel": - _, _ = section.NewKey("TYPE", "persistable-channel") - case "redis": - _, _ = section.NewKey("TYPE", "redis") - case "": - _, _ = section.NewKey("TYPE", "level") - default: - log.Fatal("Unsupported indexer queue type: %v", typ) + hasOld := false + handleOldLengthConfiguration := func(rootCfg ConfigProvider, newQueueName, oldSection, oldKey string) { + if rootCfg.Section(oldSection).HasKey(oldKey) { + hasOld = true + log.Error("Removed queue option: `[%s].%s`. Use new options in `[queue.%s]`", oldSection, oldKey, newQueueName) } } - if !directlySet.Contains("LENGTH") { - length := rootCfg.Section("indexer").Key("UPDATE_BUFFER_LEN").MustInt(0) - if length != 0 { - _, _ = section.NewKey("LENGTH", strconv.Itoa(length)) - } - } - if !directlySet.Contains("BATCH_LENGTH") { - fallback := rootCfg.Section("indexer").Key("ISSUE_INDEXER_QUEUE_BATCH_NUMBER").MustInt(0) - if fallback != 0 { - _, _ = section.NewKey("BATCH_LENGTH", strconv.Itoa(fallback)) - } - } - if !directlySet.Contains("DATADIR") { - queueDir := filepath.ToSlash(rootCfg.Section("indexer").Key("ISSUE_INDEXER_QUEUE_DIR").MustString("")) - if queueDir != "" { - _, _ = section.NewKey("DATADIR", queueDir) - } - } - if !directlySet.Contains("CONN_STR") { - connStr := rootCfg.Section("indexer").Key("ISSUE_INDEXER_QUEUE_CONN_STR").MustString("") - if connStr != "" { - _, _ = section.NewKey("CONN_STR", connStr) - } - } - - // FIXME: DEPRECATED to be removed in v1.18.0 - // - will need to set default for [queue.*)] LENGTH appropriately though though - - // Handle the old mailer configuration - handleOldLengthConfiguration(rootCfg, "mailer", "mailer", "SEND_BUFFER_LEN", 100) - - // Handle the old test pull requests configuration - // Please note this will be a unique queue - handleOldLengthConfiguration(rootCfg, "pr_patch_checker", "repository", "PULL_REQUEST_QUEUE_LENGTH", 1000) - - // Handle the old mirror queue configuration - // Please note this will be a unique queue - handleOldLengthConfiguration(rootCfg, "mirror", "repository", "MIRROR_QUEUE_LENGTH", 1000) -} - -// handleOldLengthConfiguration allows fallback to older configuration. `[queue.name]` `LENGTH` will override this configuration, but -// if that is left unset then we should fallback to the older configuration. (Except where the new length woul be <=0) -func handleOldLengthConfiguration(rootCfg ConfigProvider, queueName, oldSection, oldKey string, defaultValue int) { - if rootCfg.Section(oldSection).HasKey(oldKey) { - log.Error("Deprecated fallback for %s queue length `[%s]` `%s` present. Use `[queue.%s]` `LENGTH`. This will be removed in v1.18.0", queueName, queueName, oldSection, oldKey) - } - value := rootCfg.Section(oldSection).Key(oldKey).MustInt(defaultValue) - - // Don't override with 0 - if value <= 0 { - return - } - - section := rootCfg.Section("queue." + queueName) - directlySet := toDirectlySetKeysSet(section) - if !directlySet.Contains("LENGTH") { - _, _ = section.NewKey("LENGTH", strconv.Itoa(value)) + handleOldLengthConfiguration(rootCfg, "issue_indexer", "indexer", "ISSUE_INDEXER_QUEUE_TYPE") + handleOldLengthConfiguration(rootCfg, "issue_indexer", "indexer", "ISSUE_INDEXER_QUEUE_BATCH_NUMBER") + handleOldLengthConfiguration(rootCfg, "issue_indexer", "indexer", "ISSUE_INDEXER_QUEUE_DIR") + handleOldLengthConfiguration(rootCfg, "issue_indexer", "indexer", "ISSUE_INDEXER_QUEUE_CONN_STR") + handleOldLengthConfiguration(rootCfg, "issue_indexer", "indexer", "UPDATE_BUFFER_LEN") + handleOldLengthConfiguration(rootCfg, "mailer", "mailer", "SEND_BUFFER_LEN") + handleOldLengthConfiguration(rootCfg, "pr_patch_checker", "repository", "PULL_REQUEST_QUEUE_LENGTH") + handleOldLengthConfiguration(rootCfg, "mirror", "repository", "MIRROR_QUEUE_LENGTH") + if hasOld { + log.Fatal("Please update your app.ini to remove deprecated config options") } } - -// toDirectlySetKeysSet returns a set of keys directly set by this section -// Note: we cannot use section.HasKey(...) as that will immediately set the Key if a parent section has the Key -// but this section does not. -func toDirectlySetKeysSet(section ConfigSection) container.Set[string] { - sections := make(container.Set[string]) - for _, key := range section.Keys() { - sections.Add(key.Name()) - } - return sections -} diff --git a/modules/setting/storage_test.go b/modules/setting/storage_test.go index 9c51bbc08..5e213606e 100644 --- a/modules/setting/storage_test.go +++ b/modules/setting/storage_test.go @@ -19,7 +19,7 @@ MINIO_BUCKET = gitea-attachment STORAGE_TYPE = minio MINIO_ENDPOINT = my_minio:9000 ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) sec := cfg.Section("attachment") @@ -42,7 +42,7 @@ MINIO_BUCKET = gitea-attachment [storage.minio] MINIO_BUCKET = gitea ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) sec := cfg.Section("attachment") @@ -64,7 +64,7 @@ MINIO_BUCKET = gitea-minio [storage] MINIO_BUCKET = gitea ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) sec := cfg.Section("attachment") @@ -87,7 +87,7 @@ MINIO_BUCKET = gitea [storage] STORAGE_TYPE = local ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) sec := cfg.Section("attachment") @@ -99,7 +99,7 @@ STORAGE_TYPE = local } func Test_getStorageGetDefaults(t *testing.T) { - cfg, err := newConfigProviderFromData("") + cfg, err := NewConfigProviderFromData("") assert.NoError(t, err) sec := cfg.Section("attachment") @@ -120,7 +120,7 @@ MINIO_BUCKET = gitea-attachment [storage] MINIO_BUCKET = gitea-storage ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) { @@ -154,7 +154,7 @@ STORAGE_TYPE = lfs [storage.lfs] MINIO_BUCKET = gitea-storage ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) { @@ -178,7 +178,7 @@ func Test_getStorageInheritStorageType(t *testing.T) { [storage] STORAGE_TYPE = minio ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) sec := cfg.Section("attachment") @@ -193,7 +193,7 @@ func Test_getStorageInheritNameSectionType(t *testing.T) { [storage.attachments] STORAGE_TYPE = minio ` - cfg, err := newConfigProviderFromData(iniStr) + cfg, err := NewConfigProviderFromData(iniStr) assert.NoError(t, err) sec := cfg.Section("attachment") diff --git a/modules/test/context_tests.go b/modules/test/context_tests.go index 35dd920bb..5ba212612 100644 --- a/modules/test/context_tests.go +++ b/modules/test/context_tests.go @@ -26,6 +26,7 @@ import ( ) // MockContext mock context for unit tests +// TODO: move this function to other packages, because it depends on "models" package func MockContext(t *testing.T, path string) *context.Context { resp := &mockResponseWriter{} ctx := context.Context{ diff --git a/tests/testlogger.go b/modules/testlogger/testlogger.go similarity index 77% rename from tests/testlogger.go rename to modules/testlogger/testlogger.go index 7cac5bbe3..bf912f41d 100644 --- a/tests/testlogger.go +++ b/modules/testlogger/testlogger.go @@ -1,7 +1,7 @@ // Copyright 2019 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package tests +package testlogger import ( "context" @@ -36,56 +36,64 @@ type testLoggerWriterCloser struct { t []*testing.TB } -func (w *testLoggerWriterCloser) setT(t *testing.TB) { +func (w *testLoggerWriterCloser) pushT(t *testing.TB) { w.Lock() w.t = append(w.t, t) w.Unlock() } func (w *testLoggerWriterCloser) Write(p []byte) (int, error) { + // There was a data race problem: the logger system could still try to output logs after the runner is finished. + // So we must ensure that the "t" in stack is still valid. w.RLock() + defer w.RUnlock() + var t *testing.TB if len(w.t) > 0 { t = w.t[len(w.t)-1] } - w.RUnlock() - if t != nil && *t != nil { - if len(p) > 0 && p[len(p)-1] == '\n' { - p = p[:len(p)-1] - } - defer func() { - err := recover() - if err == nil { - return - } - var errString string - errErr, ok := err.(error) - if ok { - errString = errErr.Error() - } else { - errString, ok = err.(string) - } - if !ok { - panic(err) - } - if !strings.HasPrefix(errString, "Log in goroutine after ") { - panic(err) - } - }() - - (*t).Log(string(p)) - return len(p), nil + if len(p) > 0 && p[len(p)-1] == '\n' { + p = p[:len(p)-1] } + + if t == nil || *t == nil { + return fmt.Fprintf(os.Stdout, "??? [Unknown Test] %s\n", p) + } + + defer func() { + err := recover() + if err == nil { + return + } + var errString string + errErr, ok := err.(error) + if ok { + errString = errErr.Error() + } else { + errString, ok = err.(string) + } + if !ok { + panic(err) + } + if !strings.HasPrefix(errString, "Log in goroutine after ") { + panic(err) + } + }() + + (*t).Log(string(p)) return len(p), nil } -func (w *testLoggerWriterCloser) Close() error { +func (w *testLoggerWriterCloser) popT() { w.Lock() if len(w.t) > 0 { w.t = w.t[:len(w.t)-1] } w.Unlock() +} + +func (w *testLoggerWriterCloser) Close() error { return nil } @@ -118,7 +126,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() { } else { fmt.Fprintf(os.Stdout, "=== %s (%s:%d)\n", t.Name(), strings.TrimPrefix(filename, prefix), line) } - WriterCloser.setT(&t) + WriterCloser.pushT(&t) return func() { took := time.Since(start) if took > SlowTest { @@ -135,7 +143,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() { fmt.Fprintf(os.Stdout, "+++ %s ... still flushing after %v ...\n", t.Name(), SlowFlush) } }) - if err := queue.GetManager().FlushAll(context.Background(), 2*time.Minute); err != nil { + if err := queue.GetManager().FlushAll(context.Background(), time.Minute); err != nil { t.Errorf("Flushing queues failed with error %v", err) } timer.Stop() @@ -147,7 +155,7 @@ func PrintCurrentTest(t testing.TB, skip ...int) func() { fmt.Fprintf(os.Stdout, "+++ %s had a slow clean-up flush (took %v)\n", t.Name(), flushTook) } } - _ = WriterCloser.Close() + WriterCloser.popT() } } @@ -195,7 +203,10 @@ func (log *TestLogger) GetName() string { } func init() { - log.Register("test", NewTestLogger) + const relFilePath = "modules/testlogger/testlogger.go" _, filename, _, _ := runtime.Caller(0) - prefix = strings.TrimSuffix(filename, "tests/integration/testlogger.go") + if !strings.HasSuffix(filename, relFilePath) { + panic("source code file path doesn't match expected: " + relFilePath) + } + prefix = strings.TrimSuffix(filename, relFilePath) } diff --git a/modules/util/timer.go b/modules/util/timer.go index d598fde73..f9a795070 100644 --- a/modules/util/timer.go +++ b/modules/util/timer.go @@ -8,18 +8,6 @@ import ( "time" ) -// StopTimer is a utility function to safely stop a time.Timer and clean its channel -func StopTimer(t *time.Timer) bool { - stopped := t.Stop() - if !stopped { - select { - case <-t.C: - default: - } - } - return stopped -} - func Debounce(d time.Duration) func(f func()) { type debouncer struct { mu sync.Mutex diff --git a/routers/web/admin/admin.go b/routers/web/admin/admin.go index 35c387c28..cbe1482a2 100644 --- a/routers/web/admin/admin.go +++ b/routers/web/admin/admin.go @@ -8,13 +8,11 @@ import ( "fmt" "net/http" "runtime" - "strconv" "time" activities_model "code.gitea.io/gitea/models/activities" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/context" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/process" "code.gitea.io/gitea/modules/queue" "code.gitea.io/gitea/modules/setting" @@ -25,10 +23,10 @@ import ( ) const ( - tplDashboard base.TplName = "admin/dashboard" - tplMonitor base.TplName = "admin/monitor" - tplStacktrace base.TplName = "admin/stacktrace" - tplQueue base.TplName = "admin/queue" + tplDashboard base.TplName = "admin/dashboard" + tplMonitor base.TplName = "admin/monitor" + tplStacktrace base.TplName = "admin/stacktrace" + tplQueueManage base.TplName = "admin/queue_manage" ) var sysStatus struct { @@ -188,171 +186,3 @@ func MonitorCancel(ctx *context.Context) { "redirect": setting.AppSubURL + "/admin/monitor", }) } - -// Queue shows details for a specific queue -func Queue(ctx *context.Context) { - qid := ctx.ParamsInt64("qid") - mq := queue.GetManager().GetManagedQueue(qid) - if mq == nil { - ctx.Status(http.StatusNotFound) - return - } - ctx.Data["Title"] = ctx.Tr("admin.monitor.queue", mq.Name) - ctx.Data["PageIsAdminMonitor"] = true - ctx.Data["Queue"] = mq - ctx.HTML(http.StatusOK, tplQueue) -} - -// WorkerCancel cancels a worker group -func WorkerCancel(ctx *context.Context) { - qid := ctx.ParamsInt64("qid") - mq := queue.GetManager().GetManagedQueue(qid) - if mq == nil { - ctx.Status(http.StatusNotFound) - return - } - pid := ctx.ParamsInt64("pid") - mq.CancelWorkers(pid) - ctx.Flash.Info(ctx.Tr("admin.monitor.queue.pool.cancelling")) - ctx.JSON(http.StatusOK, map[string]interface{}{ - "redirect": setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10), - }) -} - -// Flush flushes a queue -func Flush(ctx *context.Context) { - qid := ctx.ParamsInt64("qid") - mq := queue.GetManager().GetManagedQueue(qid) - if mq == nil { - ctx.Status(http.StatusNotFound) - return - } - timeout, err := time.ParseDuration(ctx.FormString("timeout")) - if err != nil { - timeout = -1 - } - ctx.Flash.Info(ctx.Tr("admin.monitor.queue.pool.flush.added", mq.Name)) - go func() { - err := mq.Flush(timeout) - if err != nil { - log.Error("Flushing failure for %s: Error %v", mq.Name, err) - } - }() - ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) -} - -// Pause pauses a queue -func Pause(ctx *context.Context) { - qid := ctx.ParamsInt64("qid") - mq := queue.GetManager().GetManagedQueue(qid) - if mq == nil { - ctx.Status(404) - return - } - mq.Pause() - ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) -} - -// Resume resumes a queue -func Resume(ctx *context.Context) { - qid := ctx.ParamsInt64("qid") - mq := queue.GetManager().GetManagedQueue(qid) - if mq == nil { - ctx.Status(404) - return - } - mq.Resume() - ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) -} - -// AddWorkers adds workers to a worker group -func AddWorkers(ctx *context.Context) { - qid := ctx.ParamsInt64("qid") - mq := queue.GetManager().GetManagedQueue(qid) - if mq == nil { - ctx.Status(http.StatusNotFound) - return - } - number := ctx.FormInt("number") - if number < 1 { - ctx.Flash.Error(ctx.Tr("admin.monitor.queue.pool.addworkers.mustnumbergreaterzero")) - ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) - return - } - timeout, err := time.ParseDuration(ctx.FormString("timeout")) - if err != nil { - ctx.Flash.Error(ctx.Tr("admin.monitor.queue.pool.addworkers.musttimeoutduration")) - ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) - return - } - if _, ok := mq.Managed.(queue.ManagedPool); !ok { - ctx.Flash.Error(ctx.Tr("admin.monitor.queue.pool.none")) - ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) - return - } - mq.AddWorkers(number, timeout) - ctx.Flash.Success(ctx.Tr("admin.monitor.queue.pool.added")) - ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) -} - -// SetQueueSettings sets the maximum number of workers and other settings for this queue -func SetQueueSettings(ctx *context.Context) { - qid := ctx.ParamsInt64("qid") - mq := queue.GetManager().GetManagedQueue(qid) - if mq == nil { - ctx.Status(http.StatusNotFound) - return - } - if _, ok := mq.Managed.(queue.ManagedPool); !ok { - ctx.Flash.Error(ctx.Tr("admin.monitor.queue.pool.none")) - ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) - return - } - - maxNumberStr := ctx.FormString("max-number") - numberStr := ctx.FormString("number") - timeoutStr := ctx.FormString("timeout") - - var err error - var maxNumber, number int - var timeout time.Duration - if len(maxNumberStr) > 0 { - maxNumber, err = strconv.Atoi(maxNumberStr) - if err != nil { - ctx.Flash.Error(ctx.Tr("admin.monitor.queue.settings.maxnumberworkers.error")) - ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) - return - } - if maxNumber < -1 { - maxNumber = -1 - } - } else { - maxNumber = mq.MaxNumberOfWorkers() - } - - if len(numberStr) > 0 { - number, err = strconv.Atoi(numberStr) - if err != nil || number < 0 { - ctx.Flash.Error(ctx.Tr("admin.monitor.queue.settings.numberworkers.error")) - ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) - return - } - } else { - number = mq.BoostWorkers() - } - - if len(timeoutStr) > 0 { - timeout, err = time.ParseDuration(timeoutStr) - if err != nil { - ctx.Flash.Error(ctx.Tr("admin.monitor.queue.settings.timeout.error")) - ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) - return - } - } else { - timeout = mq.BoostTimeout() - } - - mq.SetPoolSettings(maxNumber, number, timeout) - ctx.Flash.Success(ctx.Tr("admin.monitor.queue.settings.changed")) - ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) -} diff --git a/routers/web/admin/queue.go b/routers/web/admin/queue.go new file mode 100644 index 000000000..1d57bc54c --- /dev/null +++ b/routers/web/admin/queue.go @@ -0,0 +1,59 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package admin + +import ( + "net/http" + "strconv" + + "code.gitea.io/gitea/modules/context" + "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/setting" +) + +// Queue shows details for a specific queue +func Queue(ctx *context.Context) { + qid := ctx.ParamsInt64("qid") + mq := queue.GetManager().GetManagedQueue(qid) + if mq == nil { + ctx.Status(http.StatusNotFound) + return + } + ctx.Data["Title"] = ctx.Tr("admin.monitor.queue", mq.GetName()) + ctx.Data["PageIsAdminMonitor"] = true + ctx.Data["Queue"] = mq + ctx.HTML(http.StatusOK, tplQueueManage) +} + +// QueueSet sets the maximum number of workers and other settings for this queue +func QueueSet(ctx *context.Context) { + qid := ctx.ParamsInt64("qid") + mq := queue.GetManager().GetManagedQueue(qid) + if mq == nil { + ctx.Status(http.StatusNotFound) + return + } + + maxNumberStr := ctx.FormString("max-number") + + var err error + var maxNumber int + if len(maxNumberStr) > 0 { + maxNumber, err = strconv.Atoi(maxNumberStr) + if err != nil { + ctx.Flash.Error(ctx.Tr("admin.monitor.queue.settings.maxnumberworkers.error")) + ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) + return + } + if maxNumber < -1 { + maxNumber = -1 + } + } else { + maxNumber = mq.GetWorkerMaxNumber() + } + + mq.SetWorkerMaxNumber(maxNumber) + ctx.Flash.Success(ctx.Tr("admin.monitor.queue.settings.changed")) + ctx.Redirect(setting.AppSubURL + "/admin/monitor/queue/" + strconv.FormatInt(qid, 10)) +} diff --git a/routers/web/web.go b/routers/web/web.go index 5917c93e2..b0db8892e 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -551,12 +551,7 @@ func registerRoutes(m *web.Route) { m.Post("/cancel/{pid}", admin.MonitorCancel) m.Group("/queue/{qid}", func() { m.Get("", admin.Queue) - m.Post("/set", admin.SetQueueSettings) - m.Post("/add", admin.AddWorkers) - m.Post("/cancel/{pid}", admin.WorkerCancel) - m.Post("/flush", admin.Flush) - m.Post("/pause", admin.Pause) - m.Post("/resume", admin.Resume) + m.Post("/set", admin.QueueSet) }) }) diff --git a/services/actions/init.go b/services/actions/init.go index 3fd03eeb6..8a9a30084 100644 --- a/services/actions/init.go +++ b/services/actions/init.go @@ -15,7 +15,7 @@ func Init() { return } - jobEmitterQueue = queue.CreateUniqueQueue("actions_ready_job", jobEmitterQueueHandle, new(jobUpdate)) + jobEmitterQueue = queue.CreateUniqueQueue("actions_ready_job", jobEmitterQueueHandler) go graceful.GetManager().RunWithShutdownFns(jobEmitterQueue.Run) notification.RegisterNotifier(NewNotifier()) diff --git a/services/actions/job_emitter.go b/services/actions/job_emitter.go index c6b6fc551..f7ec61536 100644 --- a/services/actions/job_emitter.go +++ b/services/actions/job_emitter.go @@ -16,7 +16,7 @@ import ( "xorm.io/builder" ) -var jobEmitterQueue queue.UniqueQueue +var jobEmitterQueue *queue.WorkerPoolQueue[*jobUpdate] type jobUpdate struct { RunID int64 @@ -32,13 +32,12 @@ func EmitJobsIfReady(runID int64) error { return err } -func jobEmitterQueueHandle(data ...queue.Data) []queue.Data { +func jobEmitterQueueHandler(items ...*jobUpdate) []*jobUpdate { ctx := graceful.GetManager().ShutdownContext() - var ret []queue.Data - for _, d := range data { - update := d.(*jobUpdate) + var ret []*jobUpdate + for _, update := range items { if err := checkJobsOfRun(ctx, update.RunID); err != nil { - ret = append(ret, d) + ret = append(ret, update) } } return ret diff --git a/services/automerge/automerge.go b/services/automerge/automerge.go index 994604764..f001a6ccc 100644 --- a/services/automerge/automerge.go +++ b/services/automerge/automerge.go @@ -25,11 +25,11 @@ import ( ) // prAutoMergeQueue represents a queue to handle update pull request tests -var prAutoMergeQueue queue.UniqueQueue +var prAutoMergeQueue *queue.WorkerPoolQueue[string] // Init runs the task queue to that handles auto merges func Init() error { - prAutoMergeQueue = queue.CreateUniqueQueue("pr_auto_merge", handle, "") + prAutoMergeQueue = queue.CreateUniqueQueue("pr_auto_merge", handler) if prAutoMergeQueue == nil { return fmt.Errorf("Unable to create pr_auto_merge Queue") } @@ -38,12 +38,12 @@ func Init() error { } // handle passed PR IDs and test the PRs -func handle(data ...queue.Data) []queue.Data { - for _, d := range data { +func handler(items ...string) []string { + for _, s := range items { var id int64 var sha string - if _, err := fmt.Sscanf(d.(string), "%d_%s", &id, &sha); err != nil { - log.Error("could not parse data from pr_auto_merge queue (%v): %v", d, err) + if _, err := fmt.Sscanf(s, "%d_%s", &id, &sha); err != nil { + log.Error("could not parse data from pr_auto_merge queue (%v): %v", s, err) continue } handlePull(id, sha) @@ -52,10 +52,8 @@ func handle(data ...queue.Data) []queue.Data { } func addToQueue(pr *issues_model.PullRequest, sha string) { - if err := prAutoMergeQueue.PushFunc(fmt.Sprintf("%d_%s", pr.ID, sha), func() error { - log.Trace("Adding pullID: %d to the pull requests patch checking queue with sha %s", pr.ID, sha) - return nil - }); err != nil { + log.Trace("Adding pullID: %d to the pull requests patch checking queue with sha %s", pr.ID, sha) + if err := prAutoMergeQueue.Push(fmt.Sprintf("%d_%s", pr.ID, sha)); err != nil { log.Error("Error adding pullID: %d to the pull requests patch checking queue %v", pr.ID, err) } } diff --git a/services/convert/utils_test.go b/services/convert/utils_test.go index d1ec5980c..1ac03a309 100644 --- a/services/convert/utils_test.go +++ b/services/convert/utils_test.go @@ -7,8 +7,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - - _ "github.com/mattn/go-sqlite3" ) func TestToCorrectPageSize(t *testing.T) { diff --git a/services/mailer/mailer.go b/services/mailer/mailer.go index 3d878b7c8..5aeda9ed7 100644 --- a/services/mailer/mailer.go +++ b/services/mailer/mailer.go @@ -378,7 +378,7 @@ func (s *dummySender) Send(from string, to []string, msg io.WriterTo) error { return nil } -var mailQueue queue.Queue +var mailQueue *queue.WorkerPoolQueue[*Message] // Sender sender for sending mail synchronously var Sender gomail.Sender @@ -401,9 +401,8 @@ func NewContext(ctx context.Context) { Sender = &smtpSender{} } - mailQueue = queue.CreateQueue("mail", func(data ...queue.Data) []queue.Data { - for _, datum := range data { - msg := datum.(*Message) + mailQueue = queue.CreateSimpleQueue("mail", func(items ...*Message) []*Message { + for _, msg := range items { gomailMsg := msg.ToMessage() log.Trace("New e-mail sending request %s: %s", gomailMsg.GetHeader("To"), msg.Info) if err := gomail.Send(Sender, gomailMsg); err != nil { @@ -413,7 +412,7 @@ func NewContext(ctx context.Context) { } } return nil - }, &Message{}) + }) go graceful.GetManager().RunWithShutdownFns(mailQueue.Run) diff --git a/services/migrations/github.go b/services/migrations/github.go index 26e881353..3e63fddb6 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -19,7 +19,6 @@ import ( base "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/proxy" "code.gitea.io/gitea/modules/structs" - "code.gitea.io/gitea/modules/util" "github.com/google/go-github/v51/github" "golang.org/x/oauth2" @@ -164,7 +163,7 @@ func (g *GithubDownloaderV3) waitAndPickClient() { timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time)) select { case <-g.ctx.Done(): - util.StopTimer(timer) + timer.Stop() return case <-timer.C: } diff --git a/services/mirror/mirror.go b/services/mirror/mirror.go index 9e569a70e..35ba09521 100644 --- a/services/mirror/mirror.go +++ b/services/mirror/mirror.go @@ -120,9 +120,8 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error { return nil } -func queueHandle(data ...queue.Data) []queue.Data { - for _, datum := range data { - req := datum.(*mirror_module.SyncRequest) +func queueHandler(items ...*mirror_module.SyncRequest) []*mirror_module.SyncRequest { + for _, req := range items { doMirrorSync(graceful.GetManager().ShutdownContext(), req) } return nil @@ -130,5 +129,5 @@ func queueHandle(data ...queue.Data) []queue.Data { // InitSyncMirrors initializes a go routine to sync the mirrors func InitSyncMirrors() { - mirror_module.StartSyncMirrors(queueHandle) + mirror_module.StartSyncMirrors(queueHandler) } diff --git a/services/pull/check.go b/services/pull/check.go index 02d901541..8bc2bdff1 100644 --- a/services/pull/check.go +++ b/services/pull/check.go @@ -30,7 +30,7 @@ import ( ) // prPatchCheckerQueue represents a queue to handle update pull request tests -var prPatchCheckerQueue queue.UniqueQueue +var prPatchCheckerQueue *queue.WorkerPoolQueue[string] var ( ErrIsClosed = errors.New("pull is closed") @@ -44,16 +44,14 @@ var ( // AddToTaskQueue adds itself to pull request test task queue. func AddToTaskQueue(pr *issues_model.PullRequest) { - err := prPatchCheckerQueue.PushFunc(strconv.FormatInt(pr.ID, 10), func() error { - pr.Status = issues_model.PullRequestStatusChecking - err := pr.UpdateColsIfNotMerged(db.DefaultContext, "status") - if err != nil { - log.Error("AddToTaskQueue(%-v).UpdateCols.(add to queue): %v", pr, err) - } else { - log.Trace("Adding %-v to the test pull requests queue", pr) - } - return err - }) + pr.Status = issues_model.PullRequestStatusChecking + err := pr.UpdateColsIfNotMerged(db.DefaultContext, "status") + if err != nil { + log.Error("AddToTaskQueue(%-v).UpdateCols.(add to queue): %v", pr, err) + return + } + log.Trace("Adding %-v to the test pull requests queue", pr) + err = prPatchCheckerQueue.Push(strconv.FormatInt(pr.ID, 10)) if err != nil && err != queue.ErrAlreadyInQueue { log.Error("Error adding %-v to the test pull requests queue: %v", pr, err) } @@ -315,10 +313,8 @@ func InitializePullRequests(ctx context.Context) { case <-ctx.Done(): return default: - if err := prPatchCheckerQueue.PushFunc(strconv.FormatInt(prID, 10), func() error { - log.Trace("Adding PR[%d] to the pull requests patch checking queue", prID) - return nil - }); err != nil { + log.Trace("Adding PR[%d] to the pull requests patch checking queue", prID) + if err := prPatchCheckerQueue.Push(strconv.FormatInt(prID, 10)); err != nil { log.Error("Error adding PR[%d] to the pull requests patch checking queue %v", prID, err) } } @@ -326,10 +322,9 @@ func InitializePullRequests(ctx context.Context) { } // handle passed PR IDs and test the PRs -func handle(data ...queue.Data) []queue.Data { - for _, datum := range data { - id, _ := strconv.ParseInt(datum.(string), 10, 64) - +func handler(items ...string) []string { + for _, s := range items { + id, _ := strconv.ParseInt(s, 10, 64) testPR(id) } return nil @@ -389,7 +384,7 @@ func CheckPRsForBaseBranch(baseRepo *repo_model.Repository, baseBranchName strin // Init runs the task queue to test all the checking status pull requests func Init() error { - prPatchCheckerQueue = queue.CreateUniqueQueue("pr_patch_checker", handle, "") + prPatchCheckerQueue = queue.CreateUniqueQueue("pr_patch_checker", handler) if prPatchCheckerQueue == nil { return fmt.Errorf("Unable to create pr_patch_checker Queue") diff --git a/services/pull/check_test.go b/services/pull/check_test.go index 590065250..52209b4d3 100644 --- a/services/pull/check_test.go +++ b/services/pull/check_test.go @@ -12,6 +12,7 @@ import ( issues_model "code.gitea.io/gitea/models/issues" "code.gitea.io/gitea/models/unittest" "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/setting" "github.com/stretchr/testify/assert" ) @@ -20,27 +21,18 @@ func TestPullRequest_AddToTaskQueue(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) idChan := make(chan int64, 10) - - q, err := queue.NewChannelUniqueQueue(func(data ...queue.Data) []queue.Data { - for _, datum := range data { - id, _ := strconv.ParseInt(datum.(string), 10, 64) + testHandler := func(items ...string) []string { + for _, s := range items { + id, _ := strconv.ParseInt(s, 10, 64) idChan <- id } return nil - }, queue.ChannelUniqueQueueConfiguration{ - WorkerPoolConfiguration: queue.WorkerPoolConfiguration{ - QueueLength: 10, - BatchLength: 1, - Name: "temporary-queue", - }, - Workers: 1, - }, "") + } + + cfg, err := setting.GetQueueSettings(setting.CfgProvider, "pr_patch_checker") + assert.NoError(t, err) + prPatchCheckerQueue, err = queue.NewWorkerPoolQueueBySetting("pr_patch_checker", cfg, testHandler, true) assert.NoError(t, err) - - queueShutdown := []func(){} - queueTerminate := []func(){} - - prPatchCheckerQueue = q.(queue.UniqueQueue) pr := unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{ID: 2}) AddToTaskQueue(pr) @@ -54,7 +46,8 @@ func TestPullRequest_AddToTaskQueue(t *testing.T) { assert.True(t, has) assert.NoError(t, err) - prPatchCheckerQueue.Run(func(shutdown func()) { + var queueShutdown, queueTerminate []func() + go prPatchCheckerQueue.Run(func(shutdown func()) { queueShutdown = append(queueShutdown, shutdown) }, func(terminate func()) { queueTerminate = append(queueTerminate, terminate) diff --git a/services/repository/archiver/archiver.go b/services/repository/archiver/archiver.go index 1da4425cf..1c514a411 100644 --- a/services/repository/archiver/archiver.go +++ b/services/repository/archiver/archiver.go @@ -295,26 +295,21 @@ func ArchiveRepository(request *ArchiveRequest) (*repo_model.RepoArchiver, error return doArchive(request) } -var archiverQueue queue.UniqueQueue +var archiverQueue *queue.WorkerPoolQueue[*ArchiveRequest] // Init initlize archive func Init() error { - handler := func(data ...queue.Data) []queue.Data { - for _, datum := range data { - archiveReq, ok := datum.(*ArchiveRequest) - if !ok { - log.Error("Unable to process provided datum: %v - not possible to cast to IndexerData", datum) - continue - } + handler := func(items ...*ArchiveRequest) []*ArchiveRequest { + for _, archiveReq := range items { log.Trace("ArchiverData Process: %#v", archiveReq) if _, err := doArchive(archiveReq); err != nil { - log.Error("Archive %v failed: %v", datum, err) + log.Error("Archive %v failed: %v", archiveReq, err) } } return nil } - archiverQueue = queue.CreateUniqueQueue("repo-archive", handler, new(ArchiveRequest)) + archiverQueue = queue.CreateUniqueQueue("repo-archive", handler) if archiverQueue == nil { return errors.New("unable to create codes indexer queue") } diff --git a/services/repository/push.go b/services/repository/push.go index 7f174c71b..c7ea8f336 100644 --- a/services/repository/push.go +++ b/services/repository/push.go @@ -29,12 +29,11 @@ import ( ) // pushQueue represents a queue to handle update pull request tests -var pushQueue queue.Queue +var pushQueue *queue.WorkerPoolQueue[[]*repo_module.PushUpdateOptions] // handle passed PR IDs and test the PRs -func handle(data ...queue.Data) []queue.Data { - for _, datum := range data { - opts := datum.([]*repo_module.PushUpdateOptions) +func handler(items ...[]*repo_module.PushUpdateOptions) [][]*repo_module.PushUpdateOptions { + for _, opts := range items { if err := pushUpdates(opts); err != nil { log.Error("pushUpdate failed: %v", err) } @@ -43,7 +42,7 @@ func handle(data ...queue.Data) []queue.Data { } func initPushQueue() error { - pushQueue = queue.CreateQueue("push_update", handle, []*repo_module.PushUpdateOptions{}) + pushQueue = queue.CreateSimpleQueue("push_update", handler) if pushQueue == nil { return errors.New("unable to create push_update Queue") } diff --git a/services/task/task.go b/services/task/task.go index 41bc07f2f..4f1ba3a60 100644 --- a/services/task/task.go +++ b/services/task/task.go @@ -23,7 +23,7 @@ import ( ) // taskQueue is a global queue of tasks -var taskQueue queue.Queue +var taskQueue *queue.WorkerPoolQueue[*admin_model.Task] // Run a task func Run(t *admin_model.Task) error { @@ -37,7 +37,7 @@ func Run(t *admin_model.Task) error { // Init will start the service to get all unfinished tasks and run them func Init() error { - taskQueue = queue.CreateQueue("task", handle, &admin_model.Task{}) + taskQueue = queue.CreateSimpleQueue("task", handler) if taskQueue == nil { return fmt.Errorf("Unable to create Task Queue") @@ -48,9 +48,8 @@ func Init() error { return nil } -func handle(data ...queue.Data) []queue.Data { - for _, datum := range data { - task := datum.(*admin_model.Task) +func handler(items ...*admin_model.Task) []*admin_model.Task { + for _, task := range items { if err := Run(task); err != nil { log.Error("Run task failed: %v", err) } diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go index 31246c155..e817783e5 100644 --- a/services/webhook/deliver.go +++ b/services/webhook/deliver.go @@ -283,7 +283,7 @@ func Init() error { }, } - hookQueue = queue.CreateUniqueQueue("webhook_sender", handle, int64(0)) + hookQueue = queue.CreateUniqueQueue("webhook_sender", handler) if hookQueue == nil { return fmt.Errorf("Unable to create webhook_sender Queue") } diff --git a/services/webhook/webhook.go b/services/webhook/webhook.go index b862d5bff..3cd9deafd 100644 --- a/services/webhook/webhook.go +++ b/services/webhook/webhook.go @@ -77,7 +77,7 @@ func IsValidHookTaskType(name string) bool { } // hookQueue is a global queue of web hooks -var hookQueue queue.UniqueQueue +var hookQueue *queue.WorkerPoolQueue[int64] // getPayloadBranch returns branch for hook event, if applicable. func getPayloadBranch(p api.Payloader) string { @@ -105,13 +105,13 @@ type EventSource struct { } // handle delivers hook tasks -func handle(data ...queue.Data) []queue.Data { +func handler(items ...int64) []int64 { ctx := graceful.GetManager().HammerContext() - for _, taskID := range data { - task, err := webhook_model.GetHookTaskByID(ctx, taskID.(int64)) + for _, taskID := range items { + task, err := webhook_model.GetHookTaskByID(ctx, taskID) if err != nil { - log.Error("GetHookTaskByID[%d] failed: %v", taskID.(int64), err) + log.Error("GetHookTaskByID[%d] failed: %v", taskID, err) continue } diff --git a/templates/admin/monitor.tmpl b/templates/admin/monitor.tmpl index 1ab6b75a1..6a13b7b55 100644 --- a/templates/admin/monitor.tmpl +++ b/templates/admin/monitor.tmpl @@ -1,36 +1,7 @@ {{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}}
{{template "admin/cron" .}} -

- {{.locale.Tr "admin.monitor.queues"}} -

-
- - - - - - - - - - - - - {{range .Queues}} - - - - - - - - {{end}} - -
{{.locale.Tr "admin.monitor.queue.name"}}{{.locale.Tr "admin.monitor.queue.type"}}{{.locale.Tr "admin.monitor.queue.exemplar"}}{{.locale.Tr "admin.monitor.queue.numberworkers"}}{{.locale.Tr "admin.monitor.queue.numberinqueue"}}
{{.Name}}{{.Type}}{{.ExemplarType}}{{$sum := .NumberOfWorkers}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}{{$sum = .NumberInQueue}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}{{if lt $sum 0}}{{$.locale.Tr "admin.monitor.queue.review"}}{{else}}{{$.locale.Tr "admin.monitor.queue.review_add"}}{{end}} -
-
- + {{template "admin/queue" .}} {{template "admin/process" .}}
diff --git a/templates/admin/queue.tmpl b/templates/admin/queue.tmpl index 84eb8892e..5bd507e69 100644 --- a/templates/admin/queue.tmpl +++ b/templates/admin/queue.tmpl @@ -1,192 +1,29 @@ -{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}} -
-

- {{.locale.Tr "admin.monitor.queue" .Queue.Name}} -

-
- - - - - - - - - - - - - - - - - - - - - -
{{.locale.Tr "admin.monitor.queue.name"}}{{.locale.Tr "admin.monitor.queue.type"}}{{.locale.Tr "admin.monitor.queue.exemplar"}}{{.locale.Tr "admin.monitor.queue.numberworkers"}}{{.locale.Tr "admin.monitor.queue.maxnumberworkers"}}{{.locale.Tr "admin.monitor.queue.numberinqueue"}}
{{.Queue.Name}}{{.Queue.Type}}{{.Queue.ExemplarType}}{{$sum := .Queue.NumberOfWorkers}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}{{if lt $sum 0}}-{{else}}{{.Queue.MaxNumberOfWorkers}}{{end}}{{$sum = .Queue.NumberInQueue}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}
-
- {{if lt $sum 0}} -

- {{.locale.Tr "admin.monitor.queue.nopool.title"}} -

-
- {{if eq .Queue.Type "wrapped"}} -

{{.locale.Tr "admin.monitor.queue.wrapped.desc"}}

- {{else if eq .Queue.Type "persistable-channel"}} -

{{.locale.Tr "admin.monitor.queue.persistable-channel.desc"}}

- {{else}} -

{{.locale.Tr "admin.monitor.queue.nopool.desc"}}

- {{end}} -
- {{else}} -

- {{.locale.Tr "admin.monitor.queue.settings.title"}} -

-
-

{{.locale.Tr "admin.monitor.queue.settings.desc"}}

-
- {{$.CsrfTokenHtml}} -
-
- - -
-
- - -
-
- - -
-
- - {{.locale.Tr "admin.monitor.queue.settings.blocktimeout.value" .Queue.BlockTimeout}} -
- -
-
-
-

- {{.locale.Tr "admin.monitor.queue.pool.addworkers.title"}} -

-
-

{{.locale.Tr "admin.monitor.queue.pool.addworkers.desc"}}

-
- {{$.CsrfTokenHtml}} -
-
-
- - -
-
- - -
-
- -
-
-
- {{if .Queue.Pausable}} - {{if .Queue.IsPaused}} -

- {{.locale.Tr "admin.monitor.queue.pool.resume.title"}} -

-
-

{{.locale.Tr "admin.monitor.queue.pool.resume.desc"}}

-
- {{$.CsrfTokenHtml}} -
- -
-
-
- {{else}} -

- {{.locale.Tr "admin.monitor.queue.pool.pause.title"}} -

-
-

{{.locale.Tr "admin.monitor.queue.pool.pause.desc"}}

-
- {{$.CsrfTokenHtml}} -
- -
-
-
- {{end}} +

+ {{.locale.Tr "admin.monitor.queues"}} +

+
+ + + + + + + + + + + + + {{range $qid, $q := .Queues}} + + + + + + + {{end}} -

- {{.locale.Tr "admin.monitor.queue.pool.flush.title"}} -

-
-

{{.locale.Tr "admin.monitor.queue.pool.flush.desc"}}

-
- {{$.CsrfTokenHtml}} -
-
-
- - -
-
- -
- -
-

- {{.locale.Tr "admin.monitor.queue.pool.workers.title"}} -

-
-
{{.locale.Tr "admin.monitor.queue.name"}}{{.locale.Tr "admin.monitor.queue.type"}}{{.locale.Tr "admin.monitor.queue.exemplar"}}{{.locale.Tr "admin.monitor.queue.numberworkers"}}{{.locale.Tr "admin.monitor.queue.numberinqueue"}}
{{$q.GetName}}{{$q.GetType}}{{$q.GetItemTypeName}}{{$sum := $q.GetWorkerNumber}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}{{$sum = $q.GetQueueItemNumber}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}{{if lt $sum 0}}{{$.locale.Tr "admin.monitor.queue.review"}}{{else}}{{$.locale.Tr "admin.monitor.queue.review_add"}}{{end}} +
- - - - - - - - - - {{range .Queue.Workers}} - - - - - - - {{else}} - - - {{end}} - -
{{.locale.Tr "admin.monitor.queue.numberworkers"}}{{.locale.Tr "admin.monitor.start"}}{{.locale.Tr "admin.monitor.queue.pool.timeout"}}
{{.Workers}}{{if .IsFlusher}}{{svg "octicon-sync"}}{{end}}{{DateTime "full" .Start}}{{if .HasTimeout}}{{DateTime "full" .Timeout}}{{else}}-{{end}} - {{svg "octicon-trash"}} -
{{.locale.Tr "admin.monitor.queue.pool.workers.none"}} -
-
- {{end}} -

- {{.locale.Tr "admin.monitor.queue.configuration"}} -

-
- 
{{JsonUtils.PrettyIndent .Queue.Configuration}}
-
-
- -
-
- {{.locale.Tr "admin.monitor.queue.pool.cancel"}} -
-
-

{{$.locale.Tr "admin.monitor.queue.pool.cancel_notices" `` | Safe}}

-

{{$.locale.Tr "admin.monitor.queue.pool.cancel_desc"}}

-
- {{template "base/modal_actions_confirm" .}} + +
- -{{template "admin/layout_footer" .}} diff --git a/templates/admin/queue_manage.tmpl b/templates/admin/queue_manage.tmpl new file mode 100644 index 000000000..60d6b426b --- /dev/null +++ b/templates/admin/queue_manage.tmpl @@ -0,0 +1,48 @@ +{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin monitor")}} +
+

+ {{.locale.Tr "admin.monitor.queue" .Queue.GetName}} +

+
+ + + + + + + + + + + + + + + + + + + + + +
{{.locale.Tr "admin.monitor.queue.name"}}{{.locale.Tr "admin.monitor.queue.type"}}{{.locale.Tr "admin.monitor.queue.exemplar"}}{{.locale.Tr "admin.monitor.queue.numberworkers"}}{{.locale.Tr "admin.monitor.queue.maxnumberworkers"}}{{.locale.Tr "admin.monitor.queue.numberinqueue"}}
{{.Queue.GetName}}{{.Queue.GetType}}{{.Queue.GetItemTypeName}}{{$sum := .Queue.GetWorkerNumber}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}{{if lt $sum 0}}-{{else}}{{.Queue.GetWorkerMaxNumber}}{{end}}{{$sum = .Queue.GetQueueItemNumber}}{{if lt $sum 0}}-{{else}}{{$sum}}{{end}}
+
+ +

+ {{.locale.Tr "admin.monitor.queue.settings.title"}} +

+
+

{{.locale.Tr "admin.monitor.queue.settings.desc"}}

+
+ {{$.CsrfTokenHtml}} +
+
+ + +
+ +
+
+
+
+{{template "admin/layout_footer" .}} diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go index 224b38e72..e75b3db38 100644 --- a/tests/e2e/e2e_test.go +++ b/tests/e2e/e2e_test.go @@ -21,6 +21,7 @@ import ( "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/testlogger" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers" @@ -58,7 +59,7 @@ func TestMain(m *testing.M) { exitVal := m.Run() - tests.WriterCloser.Reset() + testlogger.WriterCloser.Reset() if err = util.RemoveAll(setting.Indexer.IssuePath); err != nil { fmt.Printf("util.RemoveAll: %v\n", err) diff --git a/tests/integration/api_branch_test.go b/tests/integration/api_branch_test.go index 0d4a750a2..6616399a8 100644 --- a/tests/integration/api_branch_test.go +++ b/tests/integration/api_branch_test.go @@ -143,7 +143,6 @@ func testAPICreateBranches(t *testing.T, giteaURL *url.URL) { }, } for _, test := range testCases { - defer tests.ResetFixtures(t) session := ctx.Session testAPICreateBranch(t, session, "user2", "my-noo-repo", test.OldBranch, test.NewBranch, test.ExpectedHTTPStatus) } diff --git a/tests/integration/integration_test.go b/tests/integration/integration_test.go index fbe90ecf7..01f26d567 100644 --- a/tests/integration/integration_test.go +++ b/tests/integration/integration_test.go @@ -29,6 +29,7 @@ import ( "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/testlogger" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers" @@ -91,21 +92,21 @@ func TestMain(m *testing.M) { // integration test settings... if setting.CfgProvider != nil { testingCfg := setting.CfgProvider.Section("integration-tests") - tests.SlowTest = testingCfg.Key("SLOW_TEST").MustDuration(tests.SlowTest) - tests.SlowFlush = testingCfg.Key("SLOW_FLUSH").MustDuration(tests.SlowFlush) + testlogger.SlowTest = testingCfg.Key("SLOW_TEST").MustDuration(testlogger.SlowTest) + testlogger.SlowFlush = testingCfg.Key("SLOW_FLUSH").MustDuration(testlogger.SlowFlush) } if os.Getenv("GITEA_SLOW_TEST_TIME") != "" { duration, err := time.ParseDuration(os.Getenv("GITEA_SLOW_TEST_TIME")) if err == nil { - tests.SlowTest = duration + testlogger.SlowTest = duration } } if os.Getenv("GITEA_SLOW_FLUSH_TIME") != "" { duration, err := time.ParseDuration(os.Getenv("GITEA_SLOW_FLUSH_TIME")) if err == nil { - tests.SlowFlush = duration + testlogger.SlowFlush = duration } } @@ -130,7 +131,7 @@ func TestMain(m *testing.M) { // Instead, "No tests were found", last nonsense log is "According to the configuration, subsequent logs will not be printed to the console" exitCode := m.Run() - tests.WriterCloser.Reset() + testlogger.WriterCloser.Reset() if err = util.RemoveAll(setting.Indexer.IssuePath); err != nil { fmt.Printf("util.RemoveAll: %v\n", err) diff --git a/tests/mssql.ini.tmpl b/tests/mssql.ini.tmpl index 9cec6169f..09e75acb0 100644 --- a/tests/mssql.ini.tmpl +++ b/tests/mssql.ini.tmpl @@ -14,7 +14,7 @@ REPO_INDEXER_ENABLED = true REPO_INDEXER_PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mssql/indexers/repos.bleve [queue.issue_indexer] -PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mssql/indexers/issues.bleve +TYPE = level DATADIR = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mssql/indexers/issues.queue [queue] diff --git a/tests/mysql.ini.tmpl b/tests/mysql.ini.tmpl index b286f37bf..6fcb2792c 100644 --- a/tests/mysql.ini.tmpl +++ b/tests/mysql.ini.tmpl @@ -12,10 +12,11 @@ SSL_MODE = disable [indexer] REPO_INDEXER_ENABLED = true REPO_INDEXER_PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mysql/indexers/repos.bleve +ISSUE_INDEXER_TYPE = elasticsearch +ISSUE_INDEXER_CONN_STR = http://elastic:changeme@elasticsearch:9200 [queue.issue_indexer] -TYPE = elasticsearch -CONN_STR = http://elastic:changeme@elasticsearch:9200 +TYPE = level DATADIR = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mysql/indexers/issues.queue [queue] diff --git a/tests/mysql8.ini.tmpl b/tests/mysql8.ini.tmpl index f290efe1d..e37052da8 100644 --- a/tests/mysql8.ini.tmpl +++ b/tests/mysql8.ini.tmpl @@ -14,7 +14,7 @@ REPO_INDEXER_ENABLED = true REPO_INDEXER_PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mysql8/indexers/repos.bleve [queue.issue_indexer] -PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mysql8/indexers/issues.bleve +TYPE = level DATADIR = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mysql8/indexers/issues.queue [queue] diff --git a/tests/pgsql.ini.tmpl b/tests/pgsql.ini.tmpl index 15349aa4c..0f538797c 100644 --- a/tests/pgsql.ini.tmpl +++ b/tests/pgsql.ini.tmpl @@ -15,7 +15,7 @@ REPO_INDEXER_ENABLED = true REPO_INDEXER_PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-pgsql/indexers/repos.bleve [queue.issue_indexer] -PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-pgsql/indexers/issues.bleve +TYPE = level DATADIR = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-pgsql/indexers/issues.queue [queue] diff --git a/tests/sqlite.ini.tmpl b/tests/sqlite.ini.tmpl index d2e4a2d5a..24aff9af4 100644 --- a/tests/sqlite.ini.tmpl +++ b/tests/sqlite.ini.tmpl @@ -10,7 +10,7 @@ REPO_INDEXER_ENABLED = true REPO_INDEXER_PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-sqlite/indexers/repos.bleve [queue.issue_indexer] -PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-sqlite/indexers/issues.bleve +TYPE = level DATADIR = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-sqlite/indexers/issues.queue [queue] diff --git a/tests/test_utils.go b/tests/test_utils.go index c22b2c356..de022fde0 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -20,10 +20,10 @@ import ( "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" - "code.gitea.io/gitea/modules/queue" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" + "code.gitea.io/gitea/modules/testlogger" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/routers" @@ -61,7 +61,7 @@ func InitTest(requireGitea bool) { _ = os.Setenv("GITEA_CONF", giteaConf) fmt.Printf("Environment variable $GITEA_CONF not set, use default: %s\n", giteaConf) if !setting.EnableSQLite3 { - exitf(`Need to enable SQLite3 for sqlite.ini testing, please set: -tags "sqlite,sqlite_unlock_notify"`) + exitf(`sqlite3 requires: import _ "github.com/mattn/go-sqlite3" or -tags sqlite,sqlite_unlock_notify`) } } @@ -235,45 +235,18 @@ func PrepareTestEnv(t testing.TB, skip ...int) func() { return deferFn } -// ResetFixtures flushes queues, reloads fixtures and resets test repositories within a single test. -// Most tests should call defer tests.PrepareTestEnv(t)() (or have onGiteaRun do that for them) but sometimes -// within a single test this is required -func ResetFixtures(t *testing.T) { - assert.NoError(t, queue.GetManager().FlushAll(context.Background(), -1)) - - // load database fixtures - assert.NoError(t, unittest.LoadFixtures()) - - // load git repo fixtures - assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) - ownerDirs, err := os.ReadDir(setting.RepoRootPath) - if err != nil { - assert.NoError(t, err, "unable to read the new repo root: %v\n", err) +func PrintCurrentTest(t testing.TB, skip ...int) func() { + if len(skip) == 1 { + skip = []int{skip[0] + 1} } - for _, ownerDir := range ownerDirs { - if !ownerDir.Type().IsDir() { - continue - } - repoDirs, err := os.ReadDir(filepath.Join(setting.RepoRootPath, ownerDir.Name())) - if err != nil { - assert.NoError(t, err, "unable to read the new repo root: %v\n", err) - } - for _, repoDir := range repoDirs { - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "pack"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "objects", "info"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "heads"), 0o755) - _ = os.MkdirAll(filepath.Join(setting.RepoRootPath, ownerDir.Name(), repoDir.Name(), "refs", "tag"), 0o755) - } - } - - // load LFS object fixtures - // (LFS storage can be on any of several backends, including remote servers, so we init it with the storage API) - lfsFixtures, err := storage.NewStorage("", storage.LocalStorageConfig{Path: path.Join(filepath.Dir(setting.AppPath), "tests/gitea-lfs-meta")}) - assert.NoError(t, err) - assert.NoError(t, storage.Clean(storage.LFS)) - assert.NoError(t, lfsFixtures.IterateObjects("", func(path string, _ storage.Object) error { - _, err := storage.Copy(storage.LFS, path, lfsFixtures, path) - return err - })) + return testlogger.PrintCurrentTest(t, skip...) +} + +// Printf takes a format and args and prints the string to os.Stdout +func Printf(format string, args ...interface{}) { + testlogger.Printf(format, args...) +} + +func init() { + log.Register("test", testlogger.NewTestLogger) }