Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add default board to new projects, remove uncategorized pseudo-board #29874

Merged
merged 46 commits into from
Mar 27, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
46 commits
Select commit Hold shift + click to select a range
efaa6ce
set default board for new projects
denyskon Mar 17, 2024
069687c
Merge branch 'main' into projects-default-board
denyskon Mar 18, 2024
b82fdbd
rename uncategorized to backlog
denyskon Mar 18, 2024
5c49766
Merge branch 'main' into projects-default-board
denyskon Mar 18, 2024
53aa0da
Merge branch 'main' into projects-default-board
denyskon Mar 18, 2024
1569910
remove unchangable pseudo-board, remove setting
denyskon Mar 18, 2024
088b715
Merge branch 'projects-default-board' of ssh://github.com/denyskon/gi…
denyskon Mar 18, 2024
fc9846b
Merge branch 'main' into projects-default-board
denyskon Mar 18, 2024
8660d84
Merge branch 'main' into projects-default-board
denyskon Mar 19, 2024
98468df
fix test
denyskon Mar 19, 2024
7267f70
Merge branch 'main' into projects-default-board
denyskon Mar 19, 2024
0093855
simplify migration\n\nCo-authored-by: devlh <dev.lh@web.de>
denyskon Mar 21, 2024
d326913
Merge branch 'projects-default-board' of ssh://github.com/denyskon/gi…
denyskon Mar 21, 2024
2d5c14d
Merge branch 'main' into projects-default-board
denyskon Mar 21, 2024
d3a477a
do not insert empty board list
denyskon Mar 21, 2024
d8781f5
Merge branch 'projects-default-board' of ssh://github.com/denyskon/gi…
denyskon Mar 21, 2024
2e25dd5
Merge branch 'main' into projects-default-board
denyskon Mar 23, 2024
6f7cb7a
Update models/migrations/v1_22/v292.go
denyskon Mar 23, 2024
cbe3946
adress review comments
denyskon Mar 23, 2024
e6e5f3b
Merge branch 'main' into projects-default-board
denyskon Mar 23, 2024
5cc2f6e
Merge branch 'main' into projects-default-board
denyskon Mar 23, 2024
974a43e
add consistency check; remove migration; add test
denyskon Mar 23, 2024
7a5ab60
Merge branch 'projects-default-board' of ssh://github.com/denyskon/gi…
denyskon Mar 23, 2024
0b8fb69
Merge branch 'main' into projects-default-board
denyskon Mar 24, 2024
a3b861d
fix tests
denyskon Mar 24, 2024
42bd419
Merge branch 'main' into projects-default-board
denyskon Mar 24, 2024
add886b
fix tests, remove cronjob, run consistency check for project on getDe…
denyskon Mar 24, 2024
5541da4
Merge branch 'projects-default-board' of ssh://github.com/denyskon/gi…
denyskon Mar 24, 2024
8ea1715
Merge branch 'main' into projects-default-board
denyskon Mar 24, 2024
289cc90
make fmt
denyskon Mar 24, 2024
89c1851
Merge branch 'projects-default-board' of ssh://github.com/denyskon/gi…
denyskon Mar 24, 2024
f8b35c8
fix lint
denyskon Mar 24, 2024
b12a30a
Merge branch 'main' into projects-default-board
denyskon Mar 24, 2024
7556836
Update models/migrations/v1_22/v292.go
denyskon Mar 25, 2024
c0cf2fc
Update models/project/board.go
denyskon Mar 25, 2024
a6fcd20
Update templates/projects/view.tmpl
denyskon Mar 25, 2024
5405384
add check if board exists on setDefault
denyskon Mar 25, 2024
4e83bdb
Merge branch 'projects-default-board' of ssh://github.com/denyskon/gi…
denyskon Mar 25, 2024
e2190f6
Merge branch 'main' into projects-default-board
denyskon Mar 25, 2024
efea1c9
move board check
denyskon Mar 25, 2024
5f69ecc
Merge branch 'projects-default-board' of ssh://github.com/denyskon/gi…
denyskon Mar 25, 2024
3e330e8
Merge branch 'main' into projects-default-board
denyskon Mar 25, 2024
aa06c97
Merge remote-tracking branch 'upstream/main' into projects-default-board
denyskon Mar 27, 2024
a3b5aa0
add pagination
denyskon Mar 27, 2024
3ad1aa1
Merge branch 'projects-default-board' of ssh://github.com/denyskon/gi…
denyskon Mar 27, 2024
a2baeff
Merge branch 'main' into projects-default-board
6543 Mar 27, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions models/fixtures/project.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,27 @@
type: 2
created_unix: 1688973000
updated_unix: 1688973000

-
id: 5
title: project without default column
owner_id: 2
repo_id: 0
is_closed: false
creator_id: 2
board_type: 1
type: 2
created_unix: 1688973000
updated_unix: 1688973000

-
id: 6
title: project with multiple default columns
owner_id: 2
repo_id: 0
is_closed: false
creator_id: 2
board_type: 1
type: 2
created_unix: 1688973000
updated_unix: 1688973000
46 changes: 46 additions & 0 deletions models/fixtures/project_board.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
project_id: 1
title: To Do
creator_id: 2
default: true
created_unix: 1588117528
updated_unix: 1588117528

Expand All @@ -29,3 +30,48 @@
creator_id: 2
created_unix: 1588117528
updated_unix: 1588117528

-
id: 5
project_id: 2
title: Backlog
creator_id: 2
default: true
created_unix: 1588117528
updated_unix: 1588117528

-
id: 6
project_id: 4
title: Backlog
creator_id: 2
default: true
created_unix: 1588117528
updated_unix: 1588117528

-
id: 7
project_id: 5
title: Done
creator_id: 2
default: false
created_unix: 1588117528
updated_unix: 1588117528

-
id: 8
project_id: 6
title: Backlog
creator_id: 2
default: true
created_unix: 1588117528
updated_unix: 1588117528

-
id: 9
project_id: 6
title: Uncategorized
creator_id: 2
default: true
created_unix: 1588117528
updated_unix: 1588117528
19 changes: 7 additions & 12 deletions models/issues/issue_project.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,18 +49,13 @@ func (issue *Issue) ProjectBoardID(ctx context.Context) int64 {

// LoadIssuesFromBoard load issues assigned to this board
func LoadIssuesFromBoard(ctx context.Context, b *project_model.Board) (IssueList, error) {
issueList := make(IssueList, 0, 10)

if b.ID > 0 {
issues, err := Issues(ctx, &IssuesOptions{
ProjectBoardID: b.ID,
ProjectID: b.ProjectID,
SortType: "project-column-sorting",
})
if err != nil {
return nil, err
}
issueList = issues
issueList, err := Issues(ctx, &IssuesOptions{
ProjectBoardID: b.ID,
ProjectID: b.ProjectID,
SortType: "project-column-sorting",
})
if err != nil {
return nil, err
}

if b.Default {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
-
id: 1
title: project without default column
owner_id: 2
repo_id: 0
is_closed: false
creator_id: 2
board_type: 1
type: 2
created_unix: 1688973000
updated_unix: 1688973000

-
id: 2
title: project with multiple default columns
owner_id: 2
repo_id: 0
is_closed: false
creator_id: 2
board_type: 1
type: 2
created_unix: 1688973000
updated_unix: 1688973000
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
-
id: 1
project_id: 1
title: Done
creator_id: 2
default: false
created_unix: 1588117528
updated_unix: 1588117528

-
id: 2
project_id: 2
title: Backlog
creator_id: 2
default: true
created_unix: 1588117528
updated_unix: 1588117528

-
id: 3
project_id: 2
title: Uncategorized
creator_id: 2
default: true
created_unix: 1588117528
updated_unix: 1588117528
2 changes: 2 additions & 0 deletions models/migrations/migrations.go
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,8 @@ var migrations = []Migration{
NewMigration("Add PayloadVersion to HookTask", v1_22.AddPayloadVersionToHookTaskTable),
// v291 -> v292
NewMigration("Add Index to attachment.comment_id", v1_22.AddCommentIDIndexofAttachment),
// v292 -> v293
NewMigration("Ensure every project has exactly one default column", v1_22.CheckProjectColumnsConsistency),
}

// GetCurrentDBVersion returns the current db version
Expand Down
85 changes: 85 additions & 0 deletions models/migrations/v1_22/v292.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package v1_22 //nolint

import (
"code.gitea.io/gitea/models/project"
"code.gitea.io/gitea/modules/setting"

"xorm.io/builder"
"xorm.io/xorm"
)

// CheckProjectColumnsConsistency ensures there is exactly one default board per project present
func CheckProjectColumnsConsistency(x *xorm.Engine) error {
denyskon marked this conversation as resolved.
Show resolved Hide resolved
sess := x.NewSession()
defer sess.Close()

if err := sess.Begin(); err != nil {
return err
}

limit := setting.Database.IterateBufferSize
if limit <= 0 {
limit = 50
}

start := 0

for {
var projects []project.Project
if err := sess.SQL("SELECT DISTINCT `p`.`id`, `p`.`creator_id` FROM `project` `p` WHERE (SELECT COUNT(*) FROM `project_board` `pb` WHERE `pb`.`project_id` = `p`.`id` AND `pb`.`default` = ?) != 1", true).
Limit(limit, start).
Find(&projects); err != nil {
return err
}
Copy link
Contributor

@wxiaoguang wxiaoguang Mar 28, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry but this migration seems broken ....

If the "SELECT" results are stable and never change, this for loop is right.

BUT, by each SELECT and for-loop, the "project boards" are changed. The next SELECT, it should still process the first page results, but not skip start.

For example: 150 projects.

  1. First loop: SELECT gets 1-50 projects and updates them.
  2. Second loop: SELECT gets "the second page" from "51-150", then it gets 101-150 projects.
  3. Then the 51-100 projects are never migrated.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will send a PR to fix it.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Contributor

@melroy89 melroy89 Jun 12, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@lunny I believe I still have an issue with this migration. It says: "Ensure every project has exactly one default column". It tries to do some SQL queriy on the project and project_board tables. Migration seems to fail with error: pq: invalid input syntax for type bigint: "true". Help?? I'm stuck on migration. My docker container keeps restating in a loop now. This happens after I updated to v1.22. On version 1.21 everything worked fine.

I'm using Docker gitea/gitea:latest-rootless image with PostgreSQL v14.

2024/06/12 13:41:30 ...ations/migrations.go:642:Migrate() [I] [SQL] SELECT tablename FROM pg_tables WHERE schemaname = $1 [public] - 371.738µs
2024/06/12 13:41:30 .../xorm@v1.3.8/sync.go:30:Sync() [I] [SQL] SELECT column_name, column_default, is_nullable, data_type, character_maximum_length, description,
    CASE WHEN p.contype = 'p' THEN true ELSE false END AS primarykey,
    CASE WHEN p.contype = 'u' THEN true ELSE false END AS uniquekey
FROM pg_attribute f
    JOIN pg_class c ON c.oid = f.attrelid JOIN pg_type t ON t.oid = f.atttypid
    LEFT JOIN pg_attrdef d ON d.adrelid = c.oid AND d.adnum = f.attnum
    LEFT JOIN pg_description de ON f.attrelid=de.objoid AND f.attnum=de.objsubid
    LEFT JOIN pg_namespace n ON n.oid = c.relnamespace
    LEFT JOIN pg_constraint p ON p.conrelid = c.oid AND f.attnum = ANY (p.conkey)
    LEFT JOIN pg_class AS g ON p.confrelid = g.oid
    LEFT JOIN INFORMATION_SCHEMA.COLUMNS s ON s.column_name=f.attname AND c.relname=s.table_name
WHERE n.nspname= s.table_schema AND c.relkind = 'r' AND c.relname = $1 AND s.table_schema = $2 AND f.attnum > 0 ORDER BY f.attnum; [version public] - 2.886482ms
2024/06/12 13:41:30 .../xorm@v1.3.8/sync.go:30:Sync() [I] [SQL] SELECT indexname, indexdef FROM pg_indexes WHERE tablename=$1 AND schemaname=$2 [version public] - 431.557µs
2024/06/12 13:41:30 routers/common/db.go:31:InitDBEngine() [W] Table version column id db type is BIGINT, struct type is BIGSERIAL
2024/06/12 13:41:30 ...orm@v1.3.8/engine.go:1252:Get() [I] [SQL] SELECT "id", "version" FROM "version" WHERE "id"=$1 LIMIT 1 [1] - 223.978µs
2024/06/12 13:41:30 ...ations/migrations.go:688:Migrate() [I] Migration[293]: Ensure every project has exactly one default column
2024/06/12 13:41:30 routers/common/db.go:46:migrateWithSetting() [I] [SQL] BEGIN TRANSACTION [] - 53.6µs
2024/06/12 13:41:30 ...ations/v1_22/v293.go:55:CheckProjectColumnsConsistency() [I] [SQL] SELECT project.id as id, project.creator_id, project_board.id as board_id FROM "project" LEFT JOIN "project_board" ON project_board.project_id = project.id AND project_board."default"=$1 WHERE (project_board.id is NULL OR project_board.id = 0) LIMIT 50 [true] - 210.268µs
2024/06/12 13:41:30 ...ations/migrations.go:691:Migrate() [I] [SQL] ROLLBACK [] - 54.09µs
2024/06/12 13:41:30 routers/common/db.go:36:InitDBEngine() [E] ORM engine initialization attempt #8/10 failed. Error: migrate: migration[293]: Ensure every project has exactly one default column failed: pq: invalid input syntax for type bigint: "true"
2024/06/12 13:41:30 routers/common/db.go:37:InitDBEngine() [I] Backing off for 3 seconds
2024/06/12 13:41:31 ...eful/manager_unix.go:144:handleSignals() [W] PID 7. Received SIGTERM. Shutting down...

When I currently dump the project & project_board schema, it looks like this (hope this helps):

Project table:

CREATE TABLE "public"."project" (
    "id" bigint DEFAULT GENERATED BY DEFAULT AS IDENTITY NOT NULL,
    "title" text NOT NULL,
    "description" text,
    "repo_id" bigint,
    "creator_id" bigint NOT NULL,
    "is_closed" boolean,
    "board_type" bigint,
    "type" bigint,
    "created_unix" bigint,
    "updated_unix" bigint,
    "closed_date_unix" bigint,
    "card_type" integer DEFAULT '0' NOT NULL,
    "owner_id" bigint,
    CONSTRAINT "project_pkey" PRIMARY KEY ("id")
) WITH (oids = false);

Project board table:

CREATE TABLE "public"."project_board" (
    "id" bigint DEFAULT GENERATED BY DEFAULT AS IDENTITY NOT NULL,
    "title" text,
    "default" bigint DEFAULT '0' NOT NULL,
    "project_id" bigint NOT NULL,
    "creator_id" bigint NOT NULL,
    "created_unix" bigint,
    "updated_unix" bigint,
    "sorting" integer DEFAULT '0' NOT NULL,
    "color" character varying(7),
    CONSTRAINT "project_board_pkey" PRIMARY KEY ("id")
) WITH (oids = false);


if len(projects) == 0 {
break
}
start += len(projects)

for _, p := range projects {
var boards []project.Board
if err := sess.Where("project_id=? AND `default` = ?", p.ID, true).OrderBy("sorting").Find(&boards); err != nil {
return err
}

if len(boards) == 0 {
if _, err := sess.Insert(project.Board{
ProjectID: p.ID,
Default: true,
Title: "Uncategorized",
CreatorID: p.CreatorID,
}); err != nil {
return err
}
continue
}

var boardsToUpdate []int64
for id, b := range boards {
if id > 0 {
boardsToUpdate = append(boardsToUpdate, b.ID)
}
}

if _, err := sess.Where(builder.Eq{"project_id": p.ID}.And(builder.In("id", boardsToUpdate))).
Cols("`default`").Update(&project.Board{Default: false}); err != nil {
return err
}
}

if start%1000 == 0 {
if err := sess.Commit(); err != nil {
denyskon marked this conversation as resolved.
Show resolved Hide resolved
return err
}
if err := sess.Begin(); err != nil {
return err
}
}
}

return sess.Commit()
}
44 changes: 44 additions & 0 deletions models/migrations/v1_22/v292_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package v1_22 //nolint

import (
"testing"

"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/migrations/base"
"code.gitea.io/gitea/models/project"

"github.com/stretchr/testify/assert"
)

func Test_CheckProjectColumnsConsistency(t *testing.T) {
// Prepare and load the testing database
x, deferable := base.PrepareTestEnv(t, 0, new(project.Project), new(project.Board))
defer deferable()
if x == nil || t.Failed() {
return
}

assert.NoError(t, CheckProjectColumnsConsistency(x))

// check if default board was added
var defaultBoard project.Board
has, err := x.Where("project_id=? AND `default` = ?", 1, true).Get(&defaultBoard)
assert.NoError(t, err)
assert.True(t, has)
assert.Equal(t, int64(1), defaultBoard.ProjectID)
assert.True(t, defaultBoard.Default)

// check if multiple defaults were removed
expectDefaultBoard, err := project.GetBoard(db.DefaultContext, 2)
assert.NoError(t, err)
assert.Equal(t, int64(2), expectDefaultBoard.ProjectID)
assert.True(t, expectDefaultBoard.Default)

expectNonDefaultBoard, err := project.GetBoard(db.DefaultContext, 3)
assert.NoError(t, err)
assert.Equal(t, int64(2), expectNonDefaultBoard.ProjectID)
assert.False(t, expectNonDefaultBoard.Default)
}
Loading