From 4c8f33de278d486bc466b7813d3a43263a100a5c Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Tue, 24 Jan 2023 16:22:17 +0300 Subject: [PATCH 01/25] =?UTF-8?q?=D0=9C=D0=BD=D0=BE=D0=B3=D0=BE=D0=BF?= =?UTF-8?q?=D0=BE=D1=82=D0=BE=D1=87=D0=BD=D0=B0=D1=8F=20=D0=BE=D0=B1=D1=80?= =?UTF-8?q?=D0=B0=D0=B1=D0=BE=D1=82=D0=BA=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/config/config.go | 2 +- internal/handlers/handlers.go | 45 +++++++++- internal/handlers/handlers_test.go | 10 ++- internal/server/server.go | 2 +- internal/storage/db.go | 134 ++++++++++++++++++++++++++--- internal/storage/file.go | 22 ++++- internal/storage/sql.go | 11 ++- internal/storage/storage.go | 64 ++++++++++++-- internal/storage/storage_test.go | 41 ++++++--- 9 files changed, 281 insertions(+), 50 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 34e17ea..ca85076 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -67,4 +67,4 @@ func (c *Configuration) fillFromEnvironment() error { log.Println("Environment config:", c) return nil -} \ No newline at end of file +} diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 2f78963..6d843fb 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -14,8 +14,9 @@ import ( type Storager interface { AddURL(l, user string) (string, error) AddURLs([][2]string, string) ([][2]string, error) - FindURL(sh string) (string, error) + FindURL(sh string) (string, bool, error) GetURLsByUser(string) []string + DeleteURLs([]string, string) []string Ping() error } @@ -43,6 +44,8 @@ type PostResponseRecord struct { ShortURL string `json:"short_url"` } +type DeleteRequestBody []string + type PostRequestBatch []PostRequestRecord type PostResponseBatch []PostResponseRecord @@ -72,6 +75,7 @@ func NewHandler(s Storager, bURL string) *Handler { r.Post("/", handler.auth.Authenticate(gzipHandler(handler.postLongURL))) r.Post("/api/shorten", handler.auth.Authenticate(gzipHandler(handler.postLongURLinJSON))) r.Post("/api/shorten/batch", handler.auth.Authenticate(gzipHandler(handler.postLongURLinJSONbatch))) + r.Delete("/api/user/urls", handler.auth.Authenticate(gzipHandler(handler.deleteURLs))) r.MethodNotAllowed(handler.badRequest) }) @@ -88,14 +92,20 @@ func (h *Handler) getLongURL(w http.ResponseWriter, r *http.Request) { sh := strings.Trim(r.URL.Path, "/") log.Println("Идентификатор короткого URL, полученный из GET-запроса:", sh) - l, e := h.storage.FindURL(sh) + l, d, e := h.storage.FindURL(sh) if e != nil { log.Println("Ошибка '", e, "'. Не найден URL с указанным коротким идентификатором:", sh) http.Error(w, "URL с указанным коротким идентификатором не найден", http.StatusBadRequest) return } - log.Println("Найден URL", l, "для короткого идентификатора", sh) + if d == true { + log.Println("URL", l, "для короткого идентификатора", sh, "был удалён") + w.WriteHeader(http.StatusGone) + return + } + + log.Println("Найден URL", l, "для короткого идентификатора", sh) w.Header().Set("Location", l) w.WriteHeader(http.StatusTemporaryRedirect) } @@ -113,7 +123,7 @@ func (h *Handler) getLongURLsByUser(w http.ResponseWriter, r *http.Request) { response := make(shortAndLongURLs, 0) for i, short := range urls { - long, err := h.storage.FindURL(short) + long, _, err := h.storage.FindURL(short) if err != nil { continue } @@ -296,3 +306,30 @@ func (h *Handler) postLongURLinJSONbatch(w http.ResponseWriter, r *http.Request) log.Println("Ошибка при записи ответа в тело запроса:", e) } } + +func (h *Handler) deleteURLs(w http.ResponseWriter, r *http.Request) { + b, e := decodeRequest(r) + if e != nil { + log.Println("Неверный формат данных в запросе:", e) + http.Error(w, "неверный формат данных в запросе: "+e.Error(), http.StatusBadRequest) + return + } + + requestBody := DeleteRequestBody{} + e = json.Unmarshal(b, &requestBody) + if e != nil { + log.Println("Неверный формат данных в запросе:", e) + http.Error(w, "неверный формат данных в запросе: "+e.Error(), http.StatusBadRequest) + return + } + + if len(requestBody) == 0 { + log.Println("Пустой список идентификаторов URL") + http.Error(w, "пустой список идентификаторов URL", http.StatusBadRequest) + return + } + + _ = h.storage.DeleteURLs(requestBody, h.auth.GetUserID()) + + w.WriteHeader(http.StatusAccepted) +} diff --git a/internal/handlers/handlers_test.go b/internal/handlers/handlers_test.go index e8b2da8..c3d54a5 100644 --- a/internal/handlers/handlers_test.go +++ b/internal/handlers/handlers_test.go @@ -23,11 +23,11 @@ func (s *storage) AddURL(l, user string) (string, error) { return l, nil } -func (s *storage) FindURL(sh string) (string, error) { +func (s *storage) FindURL(sh string) (string, bool, error) { if l, ok := s.container[sh]; ok { - return l, nil + return l, false, nil } - return "", errors.New("короткий URL с ID \" + string(sh) + \" не существует") + return "", false, errors.New("короткий URL с ID \" + string(sh) + \" не существует") } func (s *storage) GetURLsByUser(u string) []string { @@ -47,6 +47,10 @@ func (s *storage) AddURLs(b [][2]string, user string) ([][2]string, error) { return b, nil } +func (s *storage) DeleteURLs(urls []string, user string) []string { + return urls +} + func TestGzipWriter_Write(t *testing.T) { t.Skip() } diff --git a/internal/server/server.go b/internal/server/server.go index 1640a11..ea8ed9a 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -13,4 +13,4 @@ func NewServer(host string, handler http.Handler) *Server { Handler: handler, }, } -} \ No newline at end of file +} diff --git a/internal/storage/db.go b/internal/storage/db.go index da0b2d8..69ce0e5 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -4,19 +4,24 @@ import ( "context" "errors" "fmt" - "github.com/jackc/pgx/v5/pgconn" - "log" - "github.com/jackc/pgerrcode" "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "log" + "sync" ) -const txPreparedName = "shurl-insert" +const txPreparedInsert = "shurl-insert" +const txPreparedDelete = "shurl-delete" type databaseStorage struct { *memoryStorage - conn *pgx.Conn - ctx context.Context + conn *pgx.Conn + ctx context.Context + deletionQueue chan string + deletionCancel context.CancelFunc + errors chan error + locker sync.Mutex } type DBError struct { @@ -24,8 +29,103 @@ type DBError struct { Err error } +const DeletionBatchSize = 20 +const DeletionQueueSize = DeletionBatchSize * 2 + +func (s *databaseStorage) DeleteURLs(shortURLs []string, user string) (deleted []string) { + deleted = make([]string, 0) + + go func() { + deleted = s.memoryStorage.DeleteURLs(shortURLs, user) + + for _, sh := range deleted { + s.deletionQueue <- sh + } + }() + + return deleted +} + +func (s *databaseStorage) deletionQueueProcess(ctx context.Context) chan error { + err := make(chan error) + + go func(s *databaseStorage, ctx context.Context) { + deletionBatch := make([]string, DeletionBatchSize) + for { + select { + case sh, ok := <-s.deletionQueue: + if !ok { + return + } + + deletionBatch = append(deletionBatch, sh) + + if len(deletionBatch) >= DeletionBatchSize { + deletionBatch = deletionBatch[:0] + } + + case <-ctx.Done(): + return + default: + } + } + }(s, ctx) + + return err +} + +func (s *databaseStorage) errorProcess() { + go func() { + for { + select { + case err, ok := <-s.errors: + if !ok { + return + } + + log.Println(err) + case <-s.ctx.Done(): + return + } + } + }() +} + +func (s *databaseStorage) delete(deletionBatch []string) { + s.locker.Lock() + defer s.locker.Unlock() + + tx, err := s.conn.Begin(s.ctx) + if err != nil { + s.errors <- err + return + } + + defer tx.Rollback(s.ctx) + + _, err = tx.Prepare(s.ctx, txPreparedDelete, queryDelete) + if err != nil { + s.errors <- err + return + } + + for _, sh := range deletionBatch { + _, err = tx.Exec(s.ctx, txPreparedDelete, sh) + if err != nil { + s.errors <- err + return + } + } + + err = tx.Commit(s.ctx) + if err != nil { + s.errors <- err + return + } +} + func newDBStorage(m *memoryStorage, database string, ctx context.Context) *databaseStorage { - storage := &databaseStorage{m, nil, ctx} + storage := &databaseStorage{memoryStorage: m, conn: nil, ctx: ctx, deletionQueue: nil} var err error storage.conn, err = pgx.Connect(ctx, database) @@ -39,6 +139,13 @@ func newDBStorage(m *memoryStorage, database string, ctx context.Context) *datab log.Fatal(err) } + storage.deletionQueue = make(chan string, DeletionQueueSize) + + var deletionCtx context.Context + deletionCtx, storage.deletionCancel = context.WithCancel(ctx) + + storage.errors = storage.deletionQueueProcess(deletionCtx) + return storage } @@ -58,12 +165,13 @@ func (s *databaseStorage) init() error { for rows.Next() { var sh, l, u string - err = rows.Scan(&sh, &l, &u) + var d bool + err = rows.Scan(&sh, &l, &u, &d) if err != nil { log.Println("Ошибка чтения из БД:", err) } - s.memoryStorage.container[sh] = l + s.memoryStorage.container[sh] = memoryRecord{longURl: l, deleted: d, user: u} s.memoryStorage.usersURLs[u] = append(s.memoryStorage.usersURLs[u], sh) } @@ -132,7 +240,7 @@ func (s *databaseStorage) AddURLs(longURLs batchURLs, user string) (batchURLs, e defer tx.Rollback(s.ctx) - _, err = tx.Prepare(s.ctx, txPreparedName, queryInsert) + _, err = tx.Prepare(s.ctx, txPreparedInsert, queryInsert) if err != nil { return result[:0], err } @@ -146,7 +254,7 @@ func (s *databaseStorage) AddURLs(longURLs batchURLs, user string) (batchURLs, e return result[:0], err } - _, err = tx.Exec(s.ctx, txPreparedName, sh, l, user) + _, err = tx.Exec(s.ctx, txPreparedInsert, sh, l, user) if err != nil { return result[:0], err } @@ -164,6 +272,10 @@ func (s *databaseStorage) AddURLs(longURLs batchURLs, user string) (batchURLs, e func (s *databaseStorage) CloseFunc() func() { return func() { + s.deletionCancel() + close(s.deletionQueue) + close(s.errors) + if s.conn == nil { return } diff --git a/internal/storage/file.go b/internal/storage/file.go index 1a71ad7..e33ced5 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -16,6 +16,7 @@ type fileStorage struct { type Record struct { ShortURL string `json:"short_url"` LongURL string `json:"long_url"` + Deleted bool `json:"deleted,omitempty"` UserID string `json:"user_id"` } @@ -68,7 +69,7 @@ func (s *fileStorage) loadFromFile() error { if r.ShortURL == "" || r.LongURL == "" { continue } - s.container[r.ShortURL] = r.LongURL + s.container[r.ShortURL] = memoryRecord{longURl: r.LongURL, deleted: r.Deleted, user: r.UserID} if r.UserID == "" { continue @@ -98,7 +99,7 @@ func (s *fileStorage) AddURL(l, user string) (string, error) { return "", err } - err = s.saveToFile(&Record{sh, l, user}) + err = s.saveToFile(&Record{ShortURL: sh, LongURL: l, Deleted: false, UserID: user}) if err != nil { return sh, err } @@ -117,7 +118,7 @@ func (s *fileStorage) AddURLs(longURLs batchURLs, user string) (batchURLs, error return result[:0], err } - err = s.saveToFile(&Record{sh, l, user}) + err = s.saveToFile(&Record{ShortURL: sh, LongURL: l, Deleted: false, UserID: user}) if err != nil { return result[:0], err } @@ -128,6 +129,19 @@ func (s *fileStorage) AddURLs(longURLs batchURLs, user string) (batchURLs, error return result, nil } +func (s *fileStorage) DeleteURLs(shortURLs []string, user string) (deleted []string) { + deleted = s.memoryStorage.DeleteURLs(shortURLs, user) + + for _, sh := range deleted { + err := s.saveToFile(&Record{ShortURL: sh, LongURL: s.container[sh].longURl, Deleted: true, UserID: user}) + if err != nil { + log.Println("Ошибка при записи удалённой ссылки с id", sh, "в файл") + } + } + + return deleted +} + func (s *fileStorage) CloseFunc() func() { return func() { if s.file == nil { @@ -142,4 +156,4 @@ func (s *fileStorage) CloseFunc() func() { log.Println("файл", s.file.Name(), "был успешно закрыт") } -} \ No newline at end of file +} diff --git a/internal/storage/sql.go b/internal/storage/sql.go index a4e2289..3d4f6aa 100644 --- a/internal/storage/sql.go +++ b/internal/storage/sql.go @@ -14,19 +14,24 @@ const ( short_url character varying(14) COLLATE pg_catalog."default" NOT NULL, long_url character varying COLLATE pg_catalog."default" NOT NULL, user_id character varying COLLATE pg_catalog."default", + deleted boolean NOT NULL DEFAULT false, + CONSTRAINT short_urls_pkey PRIMARY KEY (short_url) ) TABLESPACE pg_default; CREATE UNIQUE INDEX IF NOT EXISTS unique_long_url ON public.short_urls USING btree - (long_url COLLATE pg_catalog."default" ASC NULLS LAST) + ( long_url COLLATE pg_catalog."default" ASC NULLS LAST, + deleted ASC NULLS LAST ) TABLESPACE pg_default; ` querySelectAll = ` - SELECT short_url, long_url, user_id + SELECT short_url, long_url, user_id, deleted FROM short_urls` - querySelectByLongURL = `SELECT short_url FROM short_urls WHERE long_url = $1` + querySelectByLongURL = `SELECT short_url FROM short_urls WHERE long_url = $1 AND deleted <> true` + + queryDelete = `UPDATE short_urls SET deleted = true WHERE short_url = $1` ) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 9781d08..7af194d 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -7,6 +7,7 @@ import ( "errors" "log" "strconv" + "sync" "time" ) @@ -15,15 +16,23 @@ type batchURLs = [][2]string type Storager interface { AddURL(string, string) (string, error) AddURLs(batchURLs, string) (batchURLs, error) - FindURL(string) (string, error) + FindURL(string) (string, bool, error) GetURLsByUser(string) []string + DeleteURLs([]string, string) []string CloseFunc() func() Ping() error } +type memoryRecord struct { + longURl string + user string + deleted bool +} + type memoryStorage struct { - container map[string]string + container map[string]memoryRecord usersURLs map[string][]string + locker sync.RWMutex } func NewStorage(filePath string, database string, ctx context.Context) Storager { @@ -39,7 +48,7 @@ func NewStorage(filePath string, database string, ctx context.Context) Storager } func newMemoryStorage() *memoryStorage { - return &memoryStorage{map[string]string{}, map[string][]string{}} + return &memoryStorage{map[string]memoryRecord{}, map[string][]string{}, sync.RWMutex{}} } func generateShortURL() (string, error) { @@ -59,6 +68,9 @@ func generateShortURL() (string, error) { } func (s *memoryStorage) AddURL(l, user string) (string, error) { + s.locker.Lock() + defer s.locker.Unlock() + sh, err := generateShortURL() if err != nil { return "", err @@ -68,12 +80,15 @@ func (s *memoryStorage) AddURL(l, user string) (string, error) { return "", errors.New("короткий URL с ID " + string(sh) + " уже существует") } - s.container[sh] = l + s.container[sh] = memoryRecord{longURl: l, deleted: false, user: user} s.usersURLs[user] = append(s.usersURLs[user], sh) return sh, nil } func (s *memoryStorage) AddURLs(longURLs batchURLs, user string) (batchURLs, error) { + s.locker.Lock() + defer s.locker.Unlock() + result := make(batchURLs, 0, len(longURLs)) for _, longURL := range longURLs { id := longURL[0] @@ -90,22 +105,53 @@ func (s *memoryStorage) AddURLs(longURLs batchURLs, user string) (batchURLs, err return result, nil } -func (s *memoryStorage) FindURL(sh string) (string, error) { - if l, ok := s.container[sh]; ok { - return l, nil +func (s *memoryStorage) FindURL(sh string) (string, bool, error) { + s.locker.RLock() + defer s.locker.RUnlock() + + r, ok := s.container[sh] + if !ok { + return "", false, errors.New("короткий URL с ID \" + string(sh) + \" не существует") } - return "", errors.New("короткий URL с ID \" + string(sh) + \" не существует") + if r.deleted == true { + return "", r.deleted, errors.New("короткий URL с ID \" + string(sh) + \" удалён") + } + + return r.longURl, r.deleted, nil } func (s *memoryStorage) GetURLsByUser(u string) []string { return s.usersURLs[u] } +func (s *memoryStorage) DeleteURLs(shortURLs []string, user string) (deleted []string) { + deleted = make([]string, 0) + + s.locker.Lock() + defer s.locker.Unlock() + + for _, sh := range shortURLs { + mr, ok := s.container[sh] + if !ok { + continue + } + + if mr.user != user { + continue + } + + s.container[sh] = memoryRecord{longURl: mr.longURl, user: mr.user, deleted: true} + deleted = append(deleted, sh) + } + + return deleted +} + func (s *memoryStorage) CloseFunc() func() { return nil } func (s *memoryStorage) Ping() error { return errors.New("БД не была подключена, используется хранилище в памяти") -} \ No newline at end of file +} diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index 71e11e0..eeb5f22 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -1,6 +1,7 @@ package storage import ( + "sync" "testing" "github.com/stretchr/testify/assert" @@ -66,7 +67,7 @@ func Test_newFileStorage(t *testing.T) { func Test_memoryStorage_AddURL(t *testing.T) { tests := []struct { name string - s memoryStorage + s *memoryStorage URL string user string iterations int @@ -74,7 +75,7 @@ func Test_memoryStorage_AddURL(t *testing.T) { }{ { "Успешное добавление 1 элемента", - memoryStorage{map[string]string{}, map[string][]string{}}, + &memoryStorage{map[string]memoryRecord{}, map[string][]string{}, sync.RWMutex{}}, "http://ya.ru", "1111122222", 1, @@ -82,7 +83,7 @@ func Test_memoryStorage_AddURL(t *testing.T) { }, { "Успешное добавление дублирующих элементов", - memoryStorage{map[string]string{}, map[string][]string{}}, + &memoryStorage{map[string]memoryRecord{}, map[string][]string{}, sync.RWMutex{}}, "http://ya.ru", "3333344444", 3, @@ -103,35 +104,39 @@ func Test_memoryStorage_AddURL(t *testing.T) { func Test_memoryStorage_FindURL(t *testing.T) { tests := []struct { name string - s memoryStorage + s *memoryStorage URL string wantURL string OK bool }{ { "Неуспешная попытка поиска в пустом хранилище", - memoryStorage{map[string]string{}, map[string][]string{}}, + &memoryStorage{map[string]memoryRecord{}, map[string][]string{}, sync.RWMutex{}}, "dummy", "", false, }, { "Успешная попытка поиска в списке из 1 элемента", - memoryStorage{map[string]string{"dummy": "http://ya.ru"}, map[string][]string{}}, + &memoryStorage{map[string]memoryRecord{"dummy": {"http://ya.ru", "", false}}, map[string][]string{}, sync.RWMutex{}}, "dummy", "http://ya.ru", true, }, { "Успешная попытка поиска в списке из 3 элементов", - memoryStorage{map[string]string{"dummy": "http://ya.ru", "dummy1": "http://mail.ru", "dummy2": "http://google.ru"}, map[string][]string{}}, + &memoryStorage{map[string]memoryRecord{ + "dummy": {"http://ya.ru", "", false}, + "dummy1": {"http://mail.ru", "", false}, + "dummy2": {"http://google.ru", "", false}, + }, map[string][]string{}, sync.RWMutex{}}, "dummy1", "http://mail.ru", true, }, { "Неуспешная попытка поиска в непустом списке", - memoryStorage{map[string]string{"dummy": "http://ya.ru"}, map[string][]string{}}, + &memoryStorage{map[string]memoryRecord{"dummy": {"http://ya.ru", "", false}}, map[string][]string{}, sync.RWMutex{}}, "dummy1", "", false, @@ -139,7 +144,7 @@ func Test_memoryStorage_FindURL(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - l, err := tt.s.FindURL(tt.URL) + l, _, err := tt.s.FindURL(tt.URL) assert.Equal(t, tt.OK, err == nil) assert.Equal(t, tt.wantURL, l) }) @@ -156,28 +161,36 @@ func Test_fileStorage_AddURL(t *testing.T) { }{ { "Неуспешная попытка поиска в пустом хранилище", - fileStorage{&memoryStorage{map[string]string{}, map[string][]string{}}, nil, nil, nil}, + fileStorage{&memoryStorage{map[string]memoryRecord{}, map[string][]string{}, sync.RWMutex{}}, nil, nil, nil}, "dummy", "", false, }, { "Успешная попытка поиска в списке из 1 элемента", - fileStorage{&memoryStorage{map[string]string{"dummy": "http://ya.ru"}, map[string][]string{}}, nil, nil, nil}, + fileStorage{&memoryStorage{map[string]memoryRecord{"dummy": {"http://ya.ru", "", false}}, map[string][]string{}, sync.RWMutex{}}, nil, nil, nil}, "dummy", "http://ya.ru", true, }, { "Успешная попытка поиска в списке из 3 элементов", - fileStorage{&memoryStorage{map[string]string{"dummy": "http://ya.ru", "dummy1": "http://mail.ru", "dummy2": "http://google.ru"}, map[string][]string{}}, nil, nil, nil}, + fileStorage{&memoryStorage{map[string]memoryRecord{ + "dummy": {"http://ya.ru", "", false}, + "dummy1": {"http://mail.ru", "", false}, + "dummy2": {"http://google.ru", "", false}, + }, + map[string][]string{}, + sync.RWMutex{}, + }, + nil, nil, nil}, "dummy1", "http://mail.ru", true, }, { "Неуспешная попытка поиска в непустом списке", - fileStorage{&memoryStorage{map[string]string{"dummy": "http://ya.ru"}, map[string][]string{}}, nil, nil, nil}, + fileStorage{&memoryStorage{map[string]memoryRecord{"dummy": {"http://ya.ru", "", false}}, map[string][]string{}, sync.RWMutex{}}, nil, nil, nil}, "dummy1", "", false, @@ -185,7 +198,7 @@ func Test_fileStorage_AddURL(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - l, err := tt.s.FindURL(tt.URL) + l, _, err := tt.s.FindURL(tt.URL) assert.Equal(t, tt.OK, err == nil) assert.Equal(t, tt.wantURL, l) }) From ef462e963d224d057bbc6b83b5d552273ddd51c0 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Tue, 24 Jan 2023 16:29:31 +0300 Subject: [PATCH 02/25] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=B4=D0=BB=D1=8F=20StaticTes?= =?UTF-8?q?t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/handlers/handlers.go | 2 +- internal/storage/db.go | 9 ++++++++- internal/storage/file.go | 4 ++-- internal/storage/storage.go | 10 +++++----- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 6d843fb..09a4ee3 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -99,7 +99,7 @@ func (h *Handler) getLongURL(w http.ResponseWriter, r *http.Request) { return } - if d == true { + if d { log.Println("URL", l, "для короткого идентификатора", sh, "был удалён") w.WriteHeader(http.StatusGone) return diff --git a/internal/storage/db.go b/internal/storage/db.go index 69ce0e5..569b4a5 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -61,12 +61,19 @@ func (s *databaseStorage) deletionQueueProcess(ctx context.Context) chan error { deletionBatch = append(deletionBatch, sh) if len(deletionBatch) >= DeletionBatchSize { + s.delete(deletionBatch) deletionBatch = deletionBatch[:0] } case <-ctx.Done(): return default: + if len(deletionBatch) == 0 { + continue + } + + s.delete(deletionBatch) + deletionBatch = deletionBatch[:0] } } }(s, ctx) @@ -171,7 +178,7 @@ func (s *databaseStorage) init() error { log.Println("Ошибка чтения из БД:", err) } - s.memoryStorage.container[sh] = memoryRecord{longURl: l, deleted: d, user: u} + s.memoryStorage.container[sh] = memoryRecord{longURL: l, deleted: d, user: u} s.memoryStorage.usersURLs[u] = append(s.memoryStorage.usersURLs[u], sh) } diff --git a/internal/storage/file.go b/internal/storage/file.go index e33ced5..2191000 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -69,7 +69,7 @@ func (s *fileStorage) loadFromFile() error { if r.ShortURL == "" || r.LongURL == "" { continue } - s.container[r.ShortURL] = memoryRecord{longURl: r.LongURL, deleted: r.Deleted, user: r.UserID} + s.container[r.ShortURL] = memoryRecord{longURL: r.LongURL, deleted: r.Deleted, user: r.UserID} if r.UserID == "" { continue @@ -133,7 +133,7 @@ func (s *fileStorage) DeleteURLs(shortURLs []string, user string) (deleted []str deleted = s.memoryStorage.DeleteURLs(shortURLs, user) for _, sh := range deleted { - err := s.saveToFile(&Record{ShortURL: sh, LongURL: s.container[sh].longURl, Deleted: true, UserID: user}) + err := s.saveToFile(&Record{ShortURL: sh, LongURL: s.container[sh].longURL, Deleted: true, UserID: user}) if err != nil { log.Println("Ошибка при записи удалённой ссылки с id", sh, "в файл") } diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 7af194d..d9140c0 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -24,7 +24,7 @@ type Storager interface { } type memoryRecord struct { - longURl string + longURL string user string deleted bool } @@ -80,7 +80,7 @@ func (s *memoryStorage) AddURL(l, user string) (string, error) { return "", errors.New("короткий URL с ID " + string(sh) + " уже существует") } - s.container[sh] = memoryRecord{longURl: l, deleted: false, user: user} + s.container[sh] = memoryRecord{longURL: l, deleted: false, user: user} s.usersURLs[user] = append(s.usersURLs[user], sh) return sh, nil } @@ -114,11 +114,11 @@ func (s *memoryStorage) FindURL(sh string) (string, bool, error) { return "", false, errors.New("короткий URL с ID \" + string(sh) + \" не существует") } - if r.deleted == true { + if r.deleted { return "", r.deleted, errors.New("короткий URL с ID \" + string(sh) + \" удалён") } - return r.longURl, r.deleted, nil + return r.longURL, r.deleted, nil } func (s *memoryStorage) GetURLsByUser(u string) []string { @@ -141,7 +141,7 @@ func (s *memoryStorage) DeleteURLs(shortURLs []string, user string) (deleted []s continue } - s.container[sh] = memoryRecord{longURl: mr.longURl, user: mr.user, deleted: true} + s.container[sh] = memoryRecord{longURL: mr.longURL, user: mr.user, deleted: true} deleted = append(deleted, sh) } From 68602d46952e839b9dc0268c675d3648c65dd8d9 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Tue, 24 Jan 2023 17:41:31 +0300 Subject: [PATCH 03/25] Update handlers.go --- internal/handlers/handlers.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 09a4ee3..d026afe 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -308,6 +308,8 @@ func (h *Handler) postLongURLinJSONbatch(w http.ResponseWriter, r *http.Request) } func (h *Handler) deleteURLs(w http.ResponseWriter, r *http.Request) { + log.Println("Обработка запроса на удаление данных") + b, e := decodeRequest(r) if e != nil { log.Println("Неверный формат данных в запросе:", e) @@ -323,6 +325,8 @@ func (h *Handler) deleteURLs(w http.ResponseWriter, r *http.Request) { return } + log.Println("Тело запроса на удаление данных:\n", requestBody) + if len(requestBody) == 0 { log.Println("Пустой список идентификаторов URL") http.Error(w, "пустой список идентификаторов URL", http.StatusBadRequest) From 9e0ea5339dd1470b85a92ea6a4b182f4ad0bc9c9 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Tue, 24 Jan 2023 18:17:19 +0300 Subject: [PATCH 04/25] Update handlers.go --- internal/handlers/handlers.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index d026afe..1e0f396 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -61,6 +61,7 @@ type shortAndLongURLs []shortAndLongURL func NewHandler(s Storager, bURL string) *Handler { baseURL = bURL + log.Println("Base URL:", baseURL) handler := &Handler{ chi.NewMux(), @@ -333,6 +334,10 @@ func (h *Handler) deleteURLs(w http.ResponseWriter, r *http.Request) { return } + for i, r := range requestBody { + requestBody[i] = strings.Replace(r, baseURL, "", -1) + } + _ = h.storage.DeleteURLs(requestBody, h.auth.GetUserID()) w.WriteHeader(http.StatusAccepted) From aecc0534de206ec990f0f1a7cccc2f4b7c54b40f Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Tue, 24 Jan 2023 18:28:40 +0300 Subject: [PATCH 05/25] Update handlers.go --- internal/handlers/handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 1e0f396..7bf533b 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -326,7 +326,7 @@ func (h *Handler) deleteURLs(w http.ResponseWriter, r *http.Request) { return } - log.Println("Тело запроса на удаление данных:\n", requestBody) + log.Fatalln("Тело запроса на удаление данных:\n", requestBody) if len(requestBody) == 0 { log.Println("Пустой список идентификаторов URL") From 630976afebf1a0ac77378377a59434290f09c6ba Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Tue, 24 Jan 2023 21:53:38 +0300 Subject: [PATCH 06/25] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BE?= =?UTF-8?q?=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/auth/auth.go | 2 +- internal/handlers/handlers.go | 4 +++- internal/storage/db.go | 6 ++++++ internal/storage/storage.go | 3 +++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index dd17676..2efc495 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -81,7 +81,7 @@ func (a *authentication) authExisting(cookie string) error { } log.Println("Cookie расшифрованы в следующие байты:", data) - if userIDLength*2 < len(cookie) { + if len(cookie) < userIDLength*2 { return errors.New("неправильная длина cookie") } id := cookie[:userIDLength*2] diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 7bf533b..63eda45 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -326,7 +326,7 @@ func (h *Handler) deleteURLs(w http.ResponseWriter, r *http.Request) { return } - log.Fatalln("Тело запроса на удаление данных:\n", requestBody) + log.Println("Тело запроса на удаление данных:\n", requestBody) if len(requestBody) == 0 { log.Println("Пустой список идентификаторов URL") @@ -338,6 +338,8 @@ func (h *Handler) deleteURLs(w http.ResponseWriter, r *http.Request) { requestBody[i] = strings.Replace(r, baseURL, "", -1) } + log.Println("Список подлежащих удалению коротких идентификаторов URL:\n", requestBody) + _ = h.storage.DeleteURLs(requestBody, h.auth.GetUserID()) w.WriteHeader(http.StatusAccepted) diff --git a/internal/storage/db.go b/internal/storage/db.go index 569b4a5..3d729d3 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -208,6 +208,9 @@ func (s *databaseStorage) AddURL(l, user string) (string, error) { return "", err } + s.locker.Lock() + defer s.locker.Unlock() + ct, err := s.conn.Exec(s.ctx, queryInsert, sh, l, user) if err != nil { var pgErr *pgconn.PgError @@ -269,6 +272,9 @@ func (s *databaseStorage) AddURLs(longURLs batchURLs, user string) (batchURLs, e result = append(result, [2]string{id, sh}) } + s.locker.Lock() + defer s.locker.Unlock() + err = tx.Commit(s.ctx) if err != nil { return result[:0], err diff --git a/internal/storage/storage.go b/internal/storage/storage.go index d9140c0..b01ab63 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -122,6 +122,9 @@ func (s *memoryStorage) FindURL(sh string) (string, bool, error) { } func (s *memoryStorage) GetURLsByUser(u string) []string { + s.locker.RLock() + defer s.locker.RUnlock() + return s.usersURLs[u] } From 63dc584a3d1e6e040afa69a44d67c4c65744e644 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Tue, 24 Jan 2023 23:16:34 +0300 Subject: [PATCH 07/25] Update storage.go --- internal/storage/storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index b01ab63..a843864 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -115,7 +115,7 @@ func (s *memoryStorage) FindURL(sh string) (string, bool, error) { } if r.deleted { - return "", r.deleted, errors.New("короткий URL с ID \" + string(sh) + \" удалён") + return "", r.deleted, nil } return r.longURL, r.deleted, nil From 76faaa6f48c656a4f8ce4ff70499a67d7057377d Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Fri, 27 Jan 2023 13:13:14 +0300 Subject: [PATCH 08/25] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=B7=D0=B0=D0=BC=D0=B5=D1=87?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/handlers/handlers.go | 137 +++++++++++++++++----------------- internal/storage/db.go | 18 ++--- 2 files changed, 76 insertions(+), 79 deletions(-) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 63eda45..97ebea1 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -93,9 +93,9 @@ func (h *Handler) getLongURL(w http.ResponseWriter, r *http.Request) { sh := strings.Trim(r.URL.Path, "/") log.Println("Идентификатор короткого URL, полученный из GET-запроса:", sh) - l, d, e := h.storage.FindURL(sh) - if e != nil { - log.Println("Ошибка '", e, "'. Не найден URL с указанным коротким идентификатором:", sh) + l, d, err := h.storage.FindURL(sh) + if err != nil { + log.Println("Ошибка '", err, "'. Не найден URL с указанным коротким идентификатором:", sh) http.Error(w, "URL с указанным коротким идентификатором не найден", http.StatusBadRequest) return } @@ -145,10 +145,10 @@ func (h *Handler) getLongURLsByUser(w http.ResponseWriter, r *http.Request) { } func (h *Handler) postLongURL(w http.ResponseWriter, r *http.Request) { - b, e := decodeRequest(r) - if e != nil { - log.Println("Неверный формат данных в запросе:", e) - http.Error(w, "неверный формат данных в запросе: "+e.Error(), http.StatusBadRequest) + b, err := decodeRequest(r) + if err != nil { + log.Println("Неверный формат данных в запросе:", err) + http.Error(w, "неверный формат данных в запросе: "+err.Error(), http.StatusBadRequest) return } @@ -161,44 +161,40 @@ func (h *Handler) postLongURL(w http.ResponseWriter, r *http.Request) { return } - var duplicateFound bool - sh, e := h.storage.AddURL(l, h.auth.GetUserID()) - if e != nil { - if !strings.Contains(e.Error(), l) { - log.Println("Ошибка '", e, "' при добавлении в БД URL:", l) - http.Error(w, "ошибка при добавлении в БД: "+e.Error(), http.StatusInternalServerError) - return - } - duplicateFound = true + sh, err := h.storage.AddURL(l, h.auth.GetUserID()) + if err != nil && !strings.Contains(err.Error(), l) { + log.Println("Ошибка '", err, "' при добавлении в БД URL:", l) + http.Error(w, "ошибка при добавлении в БД: "+err.Error(), http.StatusInternalServerError) + return } - if duplicateFound { - w.WriteHeader(http.StatusConflict) + if err != nil { log.Println("Найденный короткий идентификатор URL:", sh) + w.WriteHeader(http.StatusConflict) } else { log.Println("Созданный короткий идентификатор URL:", sh) w.WriteHeader(http.StatusCreated) } - _, e = w.Write([]byte(baseURL + sh)) - if e != nil { - log.Println("Ошибка при записи ответа в тело запроса:", e) + _, err = w.Write([]byte(baseURL + sh)) + if err != nil { + log.Println("Ошибка при записи ответа в тело запроса:", err) } } func (h *Handler) postLongURLinJSON(w http.ResponseWriter, r *http.Request) { - b, e := decodeRequest(r) - if e != nil { - log.Println("Неверный формат данных в запросе:", e) - http.Error(w, "неверный формат данных в запросе: "+e.Error(), http.StatusBadRequest) + b, err := decodeRequest(r) + if err != nil { + log.Println("Неверный формат данных в запросе:", err) + http.Error(w, "неверный формат данных в запросе: "+err.Error(), http.StatusBadRequest) return } requestBody := PostRequestBody{} - e = json.Unmarshal(b, &requestBody) - if e != nil { - log.Println("Неверный формат данных в запросе:", e) - http.Error(w, "неверный формат данных в запросе: "+e.Error(), http.StatusBadRequest) + err = json.Unmarshal(b, &requestBody) + if err != nil { + log.Println("Неверный формат данных в запросе:", err) + http.Error(w, "неверный формат данных в запросе: "+err.Error(), http.StatusBadRequest) return } @@ -210,22 +206,23 @@ func (h *Handler) postLongURLinJSON(w http.ResponseWriter, r *http.Request) { } var duplicateFound bool - sh, e := h.storage.AddURL(requestBody.URL, h.auth.GetUserID()) - if e != nil { + sh, err := h.storage.AddURL(requestBody.URL, h.auth.GetUserID()) + if err != nil && !strings.Contains(err.Error(), requestBody.URL) { + log.Println("Ошибка '", err, "' при добавлении в БД URL:", requestBody.URL) + http.Error(w, "ошибка при добавлении в БД: "+err.Error(), http.StatusInternalServerError) + return + } - if !strings.Contains(e.Error(), requestBody.URL) { - log.Println("Ошибка '", e, "' при добавлении в БД URL:", requestBody.URL) - http.Error(w, "ошибка при добавлении в БД: "+e.Error(), http.StatusInternalServerError) - return - } + if err != nil { duplicateFound = true } + log.Println("Созданный короткий идентификатор URL:", sh) - response, e := json.Marshal(PostResponseBody{baseURL + sh}) - if e != nil { - log.Println("Ошибка '", e, "' при формировании ответа:", requestBody.URL) - http.Error(w, "ошибка при при формировании ответа: "+e.Error(), http.StatusInternalServerError) + response, err := json.Marshal(PostResponseBody{baseURL + sh}) + if err != nil { + log.Println("Ошибка '", err, "' при формировании ответа:", requestBody.URL) + http.Error(w, "ошибка при при формировании ответа: "+err.Error(), http.StatusInternalServerError) return } @@ -236,9 +233,9 @@ func (h *Handler) postLongURLinJSON(w http.ResponseWriter, r *http.Request) { } else { w.WriteHeader(http.StatusCreated) } - _, e = w.Write(response) - if e != nil { - log.Println("Ошибка при записи ответа в тело запроса:", e) + _, err = w.Write(response) + if err != nil { + log.Println("Ошибка при записи ответа в тело запроса:", err) } } @@ -254,18 +251,18 @@ func (h *Handler) ping(w http.ResponseWriter, r *http.Request) { } func (h *Handler) postLongURLinJSONbatch(w http.ResponseWriter, r *http.Request) { - b, e := decodeRequest(r) - if e != nil { - log.Println("Неверный формат данных в запросе:", e) - http.Error(w, "неверный формат данных в запросе: "+e.Error(), http.StatusBadRequest) + b, err := decodeRequest(r) + if err != nil { + log.Println("Неверный формат данных в запросе:", err) + http.Error(w, "неверный формат данных в запросе: "+err.Error(), http.StatusBadRequest) return } requestBody := PostRequestBatch{} - e = json.Unmarshal(b, &requestBody) - if e != nil { - log.Println("Неверный формат данных в запросе:", e) - http.Error(w, "неверный формат данных в запросе: "+e.Error(), http.StatusBadRequest) + err = json.Unmarshal(b, &requestBody) + if err != nil { + log.Println("Неверный формат данных в запросе:", err) + http.Error(w, "неверный формат данных в запросе: "+err.Error(), http.StatusBadRequest) return } @@ -280,10 +277,10 @@ func (h *Handler) postLongURLinJSONbatch(w http.ResponseWriter, r *http.Request) longURLs = append(longURLs, [2]string{requestRecord.ID, requestRecord.URL}) } - shortURLs, e := h.storage.AddURLs(longURLs, h.auth.GetUserID()) - if e != nil { - log.Println("Ошибка '", e, "' при добавлении в БД URLs:", longURLs) - http.Error(w, "ошибка при добавлении в БД URLs: "+e.Error(), http.StatusInternalServerError) + shortURLs, err := h.storage.AddURLs(longURLs, h.auth.GetUserID()) + if err != nil { + log.Println("Ошибка '", err, "' при добавлении в БД URLs:", longURLs) + http.Error(w, "ошибка при добавлении в БД URLs: "+err.Error(), http.StatusInternalServerError) return } @@ -293,36 +290,36 @@ func (h *Handler) postLongURLinJSONbatch(w http.ResponseWriter, r *http.Request) responseBody = append(responseBody, responseRecord) } - response, e := json.Marshal(responseBody) - if e != nil { - log.Println("Ошибка '", e, "' при формировании ответа:", responseBody) - http.Error(w, "ошибка при при формировании ответа: "+e.Error(), http.StatusInternalServerError) + response, err := json.Marshal(responseBody) + if err != nil { + log.Println("Ошибка '", err, "' при формировании ответа:", responseBody) + http.Error(w, "ошибка при при формировании ответа: "+err.Error(), http.StatusInternalServerError) return } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusCreated) - _, e = w.Write(response) - if e != nil { - log.Println("Ошибка при записи ответа в тело запроса:", e) + _, err = w.Write(response) + if err != nil { + log.Println("Ошибка при записи ответа в тело запроса:", err) } } func (h *Handler) deleteURLs(w http.ResponseWriter, r *http.Request) { log.Println("Обработка запроса на удаление данных") - b, e := decodeRequest(r) - if e != nil { - log.Println("Неверный формат данных в запросе:", e) - http.Error(w, "неверный формат данных в запросе: "+e.Error(), http.StatusBadRequest) + b, err := decodeRequest(r) + if err != nil { + log.Println("Неверный формат данных в запросе:", err) + http.Error(w, "неверный формат данных в запросе: "+err.Error(), http.StatusBadRequest) return } requestBody := DeleteRequestBody{} - e = json.Unmarshal(b, &requestBody) - if e != nil { - log.Println("Неверный формат данных в запросе:", e) - http.Error(w, "неверный формат данных в запросе: "+e.Error(), http.StatusBadRequest) + err = json.Unmarshal(b, &requestBody) + if err != nil { + log.Println("Неверный формат данных в запросе:", err) + http.Error(w, "неверный формат данных в запросе: "+err.Error(), http.StatusBadRequest) return } diff --git a/internal/storage/db.go b/internal/storage/db.go index 3d729d3..29768ff 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -211,19 +211,19 @@ func (s *databaseStorage) AddURL(l, user string) (string, error) { s.locker.Lock() defer s.locker.Unlock() + var pgErr *pgconn.PgError ct, err := s.conn.Exec(s.ctx, queryInsert, sh, l, user) - if err != nil { - var pgErr *pgconn.PgError - if !errors.As(err, &pgErr) { - return "", err - } + if err != nil && !errors.As(err, &pgErr) { + return "", err + } + if err != nil && pgErr.Code != pgerrcode.UniqueViolation { log.Println("Ошибка операции с БД, код:", pgErr.Code, ", сообщение:", pgErr.Error()) + return "", err + } - if pgErr.Code != pgerrcode.UniqueViolation { - return "", err - } - + if err != nil { + log.Println("Ошибка операции с БД, код:", pgErr.Code, ", сообщение:", pgErr.Error()) duplicateErr := NewStorageDBError(l, err) r := s.conn.QueryRow(s.ctx, querySelectByLongURL, l) From 5cac0843288d334554f7bf992724e9f9e32f9c25 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Thu, 2 Feb 2023 20:50:58 +0300 Subject: [PATCH 09/25] =?UTF-8?q?=D0=9F=D1=80=D0=B0=D0=B2=D0=BA=D0=B8=20?= =?UTF-8?q?=D0=BF=D0=BE=20=D1=80=D0=B5=D0=B7=D1=83=D0=BB=D1=8C=D1=82=D0=B0?= =?UTF-8?q?=D1=82=D0=B0=D0=BC=20Code=20Review?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/handlers/handlers.go | 21 +++---- internal/handlers/handlers_test.go | 14 +++-- internal/storage/db.go | 96 ++++++++++++++++-------------- internal/storage/file.go | 13 ++-- internal/storage/storage.go | 22 +++---- 5 files changed, 85 insertions(+), 81 deletions(-) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 97ebea1..ce59dce 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -6,23 +6,16 @@ import ( "net/http" "strings" + "github.com/StainlessSteelSnake/shurl/internal/storage" + "github.com/StainlessSteelSnake/shurl/internal/auth" "github.com/go-chi/chi/v5" ) -type Storager interface { - AddURL(l, user string) (string, error) - AddURLs([][2]string, string) ([][2]string, error) - FindURL(sh string) (string, bool, error) - GetURLsByUser(string) []string - DeleteURLs([]string, string) []string - Ping() error -} - type Handler struct { *chi.Mux - storage Storager + storage storage.Storager auth auth.Authenticator } @@ -59,7 +52,7 @@ type shortAndLongURL struct { type shortAndLongURLs []shortAndLongURL -func NewHandler(s Storager, bURL string) *Handler { +func NewHandler(s storage.Storager, bURL string) *Handler { baseURL = bURL log.Println("Base URL:", baseURL) @@ -272,9 +265,9 @@ func (h *Handler) postLongURLinJSONbatch(w http.ResponseWriter, r *http.Request) return } - var longURLs = make([][2]string, 0, len(requestBody)) + var longURLs = make(storage.BatchURLs, 0, len(requestBody)) for _, requestRecord := range requestBody { - longURLs = append(longURLs, [2]string{requestRecord.ID, requestRecord.URL}) + longURLs = append(longURLs, storage.RecordURL{ID: requestRecord.ID, URL: requestRecord.URL}) } shortURLs, err := h.storage.AddURLs(longURLs, h.auth.GetUserID()) @@ -286,7 +279,7 @@ func (h *Handler) postLongURLinJSONbatch(w http.ResponseWriter, r *http.Request) var responseBody = make(PostResponseBatch, 0, len(shortURLs)) for _, shortURL := range shortURLs { - responseRecord := PostResponseRecord{ID: shortURL[0], ShortURL: baseURL + shortURL[1]} + responseRecord := PostResponseRecord{ID: shortURL.ID, ShortURL: baseURL + shortURL.URL} responseBody = append(responseBody, responseRecord) } diff --git a/internal/handlers/handlers_test.go b/internal/handlers/handlers_test.go index c3d54a5..6e51f83 100644 --- a/internal/handlers/handlers_test.go +++ b/internal/handlers/handlers_test.go @@ -2,13 +2,15 @@ package handlers import ( "errors" - "github.com/StainlessSteelSnake/shurl/internal/auth" "io" "net/http" "net/http/httptest" "strings" "testing" + "github.com/StainlessSteelSnake/shurl/internal/auth" + . "github.com/StainlessSteelSnake/shurl/internal/storage" + "github.com/stretchr/testify/assert" ) @@ -38,10 +40,14 @@ func (s *storage) Ping() error { return nil } -func (s *storage) AddURLs(b [][2]string, user string) ([][2]string, error) { +func (s *storage) CloseFunc() func() { + return nil +} + +func (s *storage) AddURLs(b BatchURLs, user string) (BatchURLs, error) { for _, record := range b { - s.container[record[1]] = record[1] - s.usersURLs[user] = append(s.usersURLs[user], record[1]) + s.container[record.ID] = record.URL + s.usersURLs[user] = append(s.usersURLs[user], record.URL) } return b, nil diff --git a/internal/storage/db.go b/internal/storage/db.go index 29768ff..535d129 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -4,11 +4,12 @@ import ( "context" "errors" "fmt" + "log" + "sync" + "github.com/jackc/pgerrcode" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" - "log" - "sync" ) const txPreparedInsert = "shurl-insert" @@ -17,7 +18,6 @@ const txPreparedDelete = "shurl-delete" type databaseStorage struct { *memoryStorage conn *pgx.Conn - ctx context.Context deletionQueue chan string deletionCancel context.CancelFunc errors chan error @@ -47,7 +47,7 @@ func (s *databaseStorage) DeleteURLs(shortURLs []string, user string) (deleted [ } func (s *databaseStorage) deletionQueueProcess(ctx context.Context) chan error { - err := make(chan error) + errorChan := make(chan error) go func(s *databaseStorage, ctx context.Context) { deletionBatch := make([]string, DeletionBatchSize) @@ -61,7 +61,10 @@ func (s *databaseStorage) deletionQueueProcess(ctx context.Context) chan error { deletionBatch = append(deletionBatch, sh) if len(deletionBatch) >= DeletionBatchSize { - s.delete(deletionBatch) + err := s.delete(ctx, deletionBatch) + if err != nil { + errorChan <- err + } deletionBatch = deletionBatch[:0] } @@ -72,16 +75,19 @@ func (s *databaseStorage) deletionQueueProcess(ctx context.Context) chan error { continue } - s.delete(deletionBatch) + err := s.delete(ctx, deletionBatch) + if err != nil { + errorChan <- err + } deletionBatch = deletionBatch[:0] } } }(s, ctx) - return err + return errorChan } -func (s *databaseStorage) errorProcess() { +func (s *databaseStorage) errorProcess(ctx context.Context) { go func() { for { select { @@ -91,48 +97,46 @@ func (s *databaseStorage) errorProcess() { } log.Println(err) - case <-s.ctx.Done(): + case <-ctx.Done(): return } } }() } -func (s *databaseStorage) delete(deletionBatch []string) { +func (s *databaseStorage) delete(ctx context.Context, deletionBatch []string) error { s.locker.Lock() defer s.locker.Unlock() - tx, err := s.conn.Begin(s.ctx) + dbCtx, _ := context.WithCancel(ctx) + tx, err := s.conn.Begin(dbCtx) if err != nil { - s.errors <- err - return + return err } - defer tx.Rollback(s.ctx) + defer tx.Rollback(dbCtx) - _, err = tx.Prepare(s.ctx, txPreparedDelete, queryDelete) + _, err = tx.Prepare(dbCtx, txPreparedDelete, queryDelete) if err != nil { - s.errors <- err - return + return err } for _, sh := range deletionBatch { - _, err = tx.Exec(s.ctx, txPreparedDelete, sh) + _, err = tx.Exec(dbCtx, txPreparedDelete, sh) if err != nil { - s.errors <- err - return + return err } } - err = tx.Commit(s.ctx) + err = tx.Commit(dbCtx) if err != nil { - s.errors <- err - return + return err } + return nil } -func newDBStorage(m *memoryStorage, database string, ctx context.Context) *databaseStorage { - storage := &databaseStorage{memoryStorage: m, conn: nil, ctx: ctx, deletionQueue: nil} +func newDBStorage(ctx context.Context, m *memoryStorage, database string) *databaseStorage { + storage := &databaseStorage{memoryStorage: m, conn: nil, deletionQueue: nil} var err error storage.conn, err = pgx.Connect(ctx, database) @@ -141,7 +145,7 @@ func newDBStorage(m *memoryStorage, database string, ctx context.Context) *datab return storage } - err = storage.init() + err = storage.init(ctx) if err != nil { log.Fatal(err) } @@ -156,14 +160,14 @@ func newDBStorage(m *memoryStorage, database string, ctx context.Context) *datab return storage } -func (s *databaseStorage) init() error { +func (s *databaseStorage) init(ctx context.Context) error { - _, err := s.conn.Exec(s.ctx, queryCreateTable) + _, err := s.conn.Exec(ctx, queryCreateTable) if err != nil { return err } - rows, err := s.conn.Query(s.ctx, querySelectAll) + rows, err := s.conn.Query(ctx, querySelectAll) if err != nil { return err } @@ -211,8 +215,9 @@ func (s *databaseStorage) AddURL(l, user string) (string, error) { s.locker.Lock() defer s.locker.Unlock() + ctx, _ := context.WithCancel(context.Background()) var pgErr *pgconn.PgError - ct, err := s.conn.Exec(s.ctx, queryInsert, sh, l, user) + ct, err := s.conn.Exec(ctx, queryInsert, sh, l, user) if err != nil && !errors.As(err, &pgErr) { return "", err } @@ -226,7 +231,7 @@ func (s *databaseStorage) AddURL(l, user string) (string, error) { log.Println("Ошибка операции с БД, код:", pgErr.Code, ", сообщение:", pgErr.Error()) duplicateErr := NewStorageDBError(l, err) - r := s.conn.QueryRow(s.ctx, querySelectByLongURL, l) + r := s.conn.QueryRow(ctx, querySelectByLongURL, l) err = r.Scan(&sh) if err != nil { return "", NewStorageDBError(l, err) @@ -240,42 +245,40 @@ func (s *databaseStorage) AddURL(l, user string) (string, error) { return sh, nil } -func (s *databaseStorage) AddURLs(longURLs batchURLs, user string) (batchURLs, error) { - result := make(batchURLs, 0, len(longURLs)) +func (s *databaseStorage) AddURLs(longURLs BatchURLs, user string) (BatchURLs, error) { + result := make(BatchURLs, 0, len(longURLs)) - tx, err := s.conn.Begin(s.ctx) + ctx, _ := context.WithCancel(context.Background()) + tx, err := s.conn.Begin(ctx) if err != nil { return result[:0], err } - defer tx.Rollback(s.ctx) + defer tx.Rollback(ctx) - _, err = tx.Prepare(s.ctx, txPreparedInsert, queryInsert) + _, err = tx.Prepare(ctx, txPreparedInsert, queryInsert) if err != nil { return result[:0], err } for _, longURL := range longURLs { - id := longURL[0] - l := longURL[1] - - sh, err := s.memoryStorage.AddURL(l, user) + sh, err := s.memoryStorage.AddURL(longURL.URL, user) if err != nil { return result[:0], err } - _, err = tx.Exec(s.ctx, txPreparedInsert, sh, l, user) + _, err = tx.Exec(ctx, txPreparedInsert, sh, longURL.URL, user) if err != nil { return result[:0], err } - result = append(result, [2]string{id, sh}) + result = append(result, RecordURL{ID: longURL.ID, URL: sh}) } s.locker.Lock() defer s.locker.Unlock() - err = tx.Commit(s.ctx) + err = tx.Commit(ctx) if err != nil { return result[:0], err } @@ -293,7 +296,8 @@ func (s *databaseStorage) CloseFunc() func() { return } - err := s.conn.Close(s.ctx) + ctx, _ := context.WithCancel(context.Background()) + err := s.conn.Close(ctx) if err != nil { log.Println(err) return @@ -305,5 +309,7 @@ func (s *databaseStorage) Ping() error { if s.conn == nil { return s.memoryStorage.Ping() } - return s.conn.Ping(s.ctx) + + ctx, _ := context.WithCancel(context.Background()) + return s.conn.Ping(ctx) } diff --git a/internal/storage/file.go b/internal/storage/file.go index 2191000..6ce6860 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -107,23 +107,20 @@ func (s *fileStorage) AddURL(l, user string) (string, error) { return sh, nil } -func (s *fileStorage) AddURLs(longURLs batchURLs, user string) (batchURLs, error) { - result := make(batchURLs, 0, len(longURLs)) +func (s *fileStorage) AddURLs(longURLs BatchURLs, user string) (BatchURLs, error) { + result := make(BatchURLs, 0, len(longURLs)) for _, longURL := range longURLs { - id := longURL[0] - l := longURL[1] - - sh, err := s.AddURL(l, user) + sh, err := s.AddURL(longURL.URL, user) if err != nil { return result[:0], err } - err = s.saveToFile(&Record{ShortURL: sh, LongURL: l, Deleted: false, UserID: user}) + err = s.saveToFile(&Record{ShortURL: sh, LongURL: longURL.URL, Deleted: false, UserID: user}) if err != nil { return result[:0], err } - result = append(result, [2]string{id, sh}) + result = append(result, RecordURL{ID: longURL.ID, URL: sh}) } return result, nil diff --git a/internal/storage/storage.go b/internal/storage/storage.go index a843864..8af4690 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -11,11 +11,16 @@ import ( "time" ) -type batchURLs = [][2]string +type RecordURL struct { + ID string + URL string +} + +type BatchURLs = []RecordURL type Storager interface { AddURL(string, string) (string, error) - AddURLs(batchURLs, string) (batchURLs, error) + AddURLs(BatchURLs, string) (BatchURLs, error) FindURL(string) (string, bool, error) GetURLsByUser(string) []string DeleteURLs([]string, string) []string @@ -37,7 +42,7 @@ type memoryStorage struct { func NewStorage(filePath string, database string, ctx context.Context) Storager { if database != "" { - return newDBStorage(newMemoryStorage(), database, ctx) + return newDBStorage(ctx, newMemoryStorage(), database) } if filePath != "" { @@ -85,21 +90,18 @@ func (s *memoryStorage) AddURL(l, user string) (string, error) { return sh, nil } -func (s *memoryStorage) AddURLs(longURLs batchURLs, user string) (batchURLs, error) { +func (s *memoryStorage) AddURLs(longURLs BatchURLs, user string) (BatchURLs, error) { s.locker.Lock() defer s.locker.Unlock() - result := make(batchURLs, 0, len(longURLs)) + result := make(BatchURLs, 0, len(longURLs)) for _, longURL := range longURLs { - id := longURL[0] - l := longURL[1] - - sh, err := s.AddURL(l, user) + sh, err := s.AddURL(longURL.URL, user) if err != nil { return result[:0], err } - result = append(result, [2]string{id, sh}) + result = append(result, RecordURL{longURL.ID, sh}) } return result, nil From 35747a326980b6808357cfa8f2cc501300d2d8ee Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Thu, 2 Feb 2023 20:57:29 +0300 Subject: [PATCH 10/25] Update db.go --- internal/storage/db.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/internal/storage/db.go b/internal/storage/db.go index 535d129..0147e20 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -108,27 +108,26 @@ func (s *databaseStorage) delete(ctx context.Context, deletionBatch []string) er s.locker.Lock() defer s.locker.Unlock() - dbCtx, _ := context.WithCancel(ctx) - tx, err := s.conn.Begin(dbCtx) + tx, err := s.conn.Begin(ctx) if err != nil { return err } - defer tx.Rollback(dbCtx) + defer tx.Rollback(ctx) - _, err = tx.Prepare(dbCtx, txPreparedDelete, queryDelete) + _, err = tx.Prepare(ctx, txPreparedDelete, queryDelete) if err != nil { return err } for _, sh := range deletionBatch { - _, err = tx.Exec(dbCtx, txPreparedDelete, sh) + _, err = tx.Exec(ctx, txPreparedDelete, sh) if err != nil { return err } } - err = tx.Commit(dbCtx) + err = tx.Commit(ctx) if err != nil { return err } @@ -215,7 +214,7 @@ func (s *databaseStorage) AddURL(l, user string) (string, error) { s.locker.Lock() defer s.locker.Unlock() - ctx, _ := context.WithCancel(context.Background()) + ctx := context.Background() var pgErr *pgconn.PgError ct, err := s.conn.Exec(ctx, queryInsert, sh, l, user) if err != nil && !errors.As(err, &pgErr) { @@ -248,7 +247,7 @@ func (s *databaseStorage) AddURL(l, user string) (string, error) { func (s *databaseStorage) AddURLs(longURLs BatchURLs, user string) (BatchURLs, error) { result := make(BatchURLs, 0, len(longURLs)) - ctx, _ := context.WithCancel(context.Background()) + ctx := context.Background() tx, err := s.conn.Begin(ctx) if err != nil { return result[:0], err @@ -296,7 +295,7 @@ func (s *databaseStorage) CloseFunc() func() { return } - ctx, _ := context.WithCancel(context.Background()) + ctx := context.Background() err := s.conn.Close(ctx) if err != nil { log.Println(err) @@ -310,6 +309,6 @@ func (s *databaseStorage) Ping() error { return s.memoryStorage.Ping() } - ctx, _ := context.WithCancel(context.Background()) + ctx := context.Background() return s.conn.Ping(ctx) } From 12a49f597eb50509a9570e1850b274245e73f466 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Thu, 2 Feb 2023 21:00:31 +0300 Subject: [PATCH 11/25] Update handlers_test.go --- internal/handlers/handlers_test.go | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/internal/handlers/handlers_test.go b/internal/handlers/handlers_test.go index 6e51f83..f91d30d 100644 --- a/internal/handlers/handlers_test.go +++ b/internal/handlers/handlers_test.go @@ -9,42 +9,42 @@ import ( "testing" "github.com/StainlessSteelSnake/shurl/internal/auth" - . "github.com/StainlessSteelSnake/shurl/internal/storage" + "github.com/StainlessSteelSnake/shurl/internal/storage" "github.com/stretchr/testify/assert" ) -type storage struct { +type dummyStorage struct { container map[string]string usersURLs map[string][]string } -func (s *storage) AddURL(l, user string) (string, error) { +func (s *dummyStorage) AddURL(l, user string) (string, error) { s.container[l] = l s.usersURLs[user] = append(s.usersURLs[user], l) return l, nil } -func (s *storage) FindURL(sh string) (string, bool, error) { +func (s *dummyStorage) FindURL(sh string) (string, bool, error) { if l, ok := s.container[sh]; ok { return l, false, nil } return "", false, errors.New("короткий URL с ID \" + string(sh) + \" не существует") } -func (s *storage) GetURLsByUser(u string) []string { +func (s *dummyStorage) GetURLsByUser(u string) []string { return s.usersURLs[u] } -func (s *storage) Ping() error { +func (s *dummyStorage) Ping() error { return nil } -func (s *storage) CloseFunc() func() { +func (s *dummyStorage) CloseFunc() func() { return nil } -func (s *storage) AddURLs(b BatchURLs, user string) (BatchURLs, error) { +func (s *dummyStorage) AddURLs(b storage.BatchURLs, user string) (storage.BatchURLs, error) { for _, record := range b { s.container[record.ID] = record.URL s.usersURLs[user] = append(s.usersURLs[user], record.URL) @@ -53,7 +53,7 @@ func (s *storage) AddURLs(b BatchURLs, user string) (BatchURLs, error) { return b, nil } -func (s *storage) DeleteURLs(urls []string, user string) []string { +func (s *dummyStorage) DeleteURLs(urls []string, user string) []string { return urls } @@ -146,7 +146,7 @@ func TestNewHandler(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := &storage{tt.storage, tt.user} + s := &dummyStorage{tt.storage, tt.user} h := NewHandler(s, tt.baseURL) request := httptest.NewRequest(tt.method, tt.request, nil) @@ -200,7 +200,7 @@ func Test_getLongURL(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - h := Handler{storage: &storage{tt.storage, tt.user}} + h := Handler{storage: &dummyStorage{tt.storage, tt.user}} writer := httptest.NewRecorder() request := httptest.NewRequest(http.MethodGet, tt.request, nil) @@ -247,7 +247,7 @@ func Test_postLongURL(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - h := Handler{storage: &storage{tt.storage, tt.user}, auth: auth.NewAuth()} + h := Handler{storage: &dummyStorage{tt.storage, tt.user}, auth: auth.NewAuth()} writer := httptest.NewRecorder() requestBody := strings.NewReader(tt.longURL) @@ -330,7 +330,7 @@ func Test_postLongURLinJSON(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - h := Handler{storage: &storage{tt.storage, tt.user}, auth: auth.NewAuth()} + h := Handler{storage: &dummyStorage{tt.storage, tt.user}, auth: auth.NewAuth()} writer := httptest.NewRecorder() requestBody := strings.NewReader(tt.longURL) From 41f559a249a5971912624720efb8c528d4e2e851 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Mon, 6 Feb 2023 14:22:29 +0300 Subject: [PATCH 12/25] =?UTF-8?q?=D0=A7=D0=B0=D1=81=D1=82=D1=8C=20=D0=BE?= =?UTF-8?q?=D0=B1=D1=80=D0=B0=D0=B1=D0=BE=D1=82=D1=87=D0=B8=D0=BA=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=B2=D1=8B=D0=BD=D0=B5=D1=81=D0=B5=D0=BD=D1=8B=20?= =?UTF-8?q?=D0=B2=20middleware?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/auth/auth.go | 10 +++++----- internal/handlers/handlers.go | 15 +++++++++------ internal/handlers/middleware.go | 8 ++++---- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/internal/auth/auth.go b/internal/auth/auth.go index 2efc495..d3cabf7 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -21,7 +21,7 @@ type authentication struct { } type Authenticator interface { - Authenticate(http.HandlerFunc) http.HandlerFunc + Authenticate(http.Handler) http.Handler GetUserID() string } @@ -110,8 +110,8 @@ func (a *authentication) authExisting(cookie string) error { return nil } -func (a *authentication) Authenticate(next http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { +func (a *authentication) Authenticate(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if a == nil { return } @@ -143,8 +143,8 @@ func (a *authentication) Authenticate(next http.HandlerFunc) http.HandlerFunc { http.SetCookie(w, &http.Cookie{Name: cookieAuthentication, Value: a.cookieFull}) } - next(w, r) - } + next.ServeHTTP(w, r) + }) } func (a *authentication) GetUserID() string { diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index ce59dce..5fd9ce5 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -63,13 +63,16 @@ func NewHandler(s storage.Storager, bURL string) *Handler { } handler.Route("/", func(r chi.Router) { - r.Get("/{id}", handler.auth.Authenticate(gzipHandler(handler.getLongURL))) - r.Get("/api/user/urls", handler.auth.Authenticate(gzipHandler(handler.getLongURLsByUser))) + handler.Use(handler.auth.Authenticate) + handler.Use(gzipHandler) + + r.Get("/{id}", handler.getLongURL) + r.Get("/api/user/urls", handler.getLongURLsByUser) r.Get("/ping", handler.ping) - r.Post("/", handler.auth.Authenticate(gzipHandler(handler.postLongURL))) - r.Post("/api/shorten", handler.auth.Authenticate(gzipHandler(handler.postLongURLinJSON))) - r.Post("/api/shorten/batch", handler.auth.Authenticate(gzipHandler(handler.postLongURLinJSONbatch))) - r.Delete("/api/user/urls", handler.auth.Authenticate(gzipHandler(handler.deleteURLs))) + r.Post("/", handler.postLongURL) + r.Post("/api/shorten", handler.postLongURLinJSON) + r.Post("/api/shorten/batch", handler.postLongURLinJSONbatch) + r.Delete("/api/user/urls", handler.deleteURLs) r.MethodNotAllowed(handler.badRequest) }) diff --git a/internal/handlers/middleware.go b/internal/handlers/middleware.go index 860c680..1cf0489 100644 --- a/internal/handlers/middleware.go +++ b/internal/handlers/middleware.go @@ -17,11 +17,11 @@ func (w gzipWriter) Write(b []byte) (int, error) { return w.Writer.Write(b) } -func gzipHandler(next http.HandlerFunc) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { +func gzipHandler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if !strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") { log.Println("Клиент не принимает ответы в gzip") - next(w, r) + next.ServeHTTP(w, r) return } @@ -35,7 +35,7 @@ func gzipHandler(next http.HandlerFunc) http.HandlerFunc { w.Header().Set("Content-Encoding", "gzip") next.ServeHTTP(gzipWriter{ResponseWriter: w, Writer: gz}, r) - } + }) } func decodeRequest(r *http.Request) ([]byte, error) { From 72e762043e5a816a21ea8ae2865cfd7d7b4c42f3 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Mon, 6 Feb 2023 14:40:57 +0300 Subject: [PATCH 13/25] Update handlers.go --- internal/handlers/handlers.go | 50 +++++++++++++++++------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 5fd9ce5..620f264 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -86,24 +86,24 @@ func (h *Handler) badRequest(w http.ResponseWriter, r *http.Request) { func (h *Handler) getLongURL(w http.ResponseWriter, r *http.Request) { log.Println("Полученный GET-запрос:", r.URL) - sh := strings.Trim(r.URL.Path, "/") - log.Println("Идентификатор короткого URL, полученный из GET-запроса:", sh) + shortURL := strings.Trim(r.URL.Path, "/") + log.Println("Идентификатор короткого URL, полученный из GET-запроса:", shortURL) - l, d, err := h.storage.FindURL(sh) + longURL, deleted, err := h.storage.FindURL(shortURL) if err != nil { - log.Println("Ошибка '", err, "'. Не найден URL с указанным коротким идентификатором:", sh) + log.Println("Ошибка '", err, "'. Не найден URL с указанным коротким идентификатором:", shortURL) http.Error(w, "URL с указанным коротким идентификатором не найден", http.StatusBadRequest) return } - if d { - log.Println("URL", l, "для короткого идентификатора", sh, "был удалён") + if deleted { + log.Println("URL", longURL, "для короткого идентификатора", shortURL, "был удалён") w.WriteHeader(http.StatusGone) return } - log.Println("Найден URL", l, "для короткого идентификатора", sh) - w.Header().Set("Location", l) + log.Println("Найден URL", longURL, "для короткого идентификатора", shortURL) + w.Header().Set("Location", longURL) w.WriteHeader(http.StatusTemporaryRedirect) } @@ -119,13 +119,13 @@ func (h *Handler) getLongURLsByUser(w http.ResponseWriter, r *http.Request) { log.Println("Для пользователя с идентификатором '"+h.auth.GetUserID()+"' найдено ", len(urls), "сохранённых URL:") response := make(shortAndLongURLs, 0) - for i, short := range urls { - long, _, err := h.storage.FindURL(short) + for i, shortURL := range urls { + longURL, _, err := h.storage.FindURL(shortURL) if err != nil { continue } - record := shortAndLongURL{baseURL + short, long} + record := shortAndLongURL{baseURL + shortURL, longURL} log.Println("Запись", i, "короткий URL", record.ShortURL, "длинный URL", record.LongURL) response = append(response, record) } @@ -148,31 +148,31 @@ func (h *Handler) postLongURL(w http.ResponseWriter, r *http.Request) { return } - l := string(b) - log.Println("Пришедший в запросе исходный URL:", l) - if len(l) == 0 { + longURL := string(b) + log.Println("Пришедший в запросе исходный URL:", longURL) + if len(longURL) == 0 { log.Println("Неверный формат URL") http.Error(w, "неверный формат URL", http.StatusBadRequest) return } - sh, err := h.storage.AddURL(l, h.auth.GetUserID()) - if err != nil && !strings.Contains(err.Error(), l) { - log.Println("Ошибка '", err, "' при добавлении в БД URL:", l) + shortURL, err := h.storage.AddURL(longURL, h.auth.GetUserID()) + if err != nil && !strings.Contains(err.Error(), longURL) { + log.Println("Ошибка '", err, "' при добавлении в БД URL:", longURL) http.Error(w, "ошибка при добавлении в БД: "+err.Error(), http.StatusInternalServerError) return } if err != nil { - log.Println("Найденный короткий идентификатор URL:", sh) + log.Println("Найденный короткий идентификатор URL:", shortURL) w.WriteHeader(http.StatusConflict) } else { - log.Println("Созданный короткий идентификатор URL:", sh) + log.Println("Созданный короткий идентификатор URL:", shortURL) w.WriteHeader(http.StatusCreated) } - _, err = w.Write([]byte(baseURL + sh)) + _, err = w.Write([]byte(baseURL + shortURL)) if err != nil { log.Println("Ошибка при записи ответа в тело запроса:", err) } @@ -202,7 +202,7 @@ func (h *Handler) postLongURLinJSON(w http.ResponseWriter, r *http.Request) { } var duplicateFound bool - sh, err := h.storage.AddURL(requestBody.URL, h.auth.GetUserID()) + shortURL, err := h.storage.AddURL(requestBody.URL, h.auth.GetUserID()) if err != nil && !strings.Contains(err.Error(), requestBody.URL) { log.Println("Ошибка '", err, "' при добавлении в БД URL:", requestBody.URL) http.Error(w, "ошибка при добавлении в БД: "+err.Error(), http.StatusInternalServerError) @@ -213,9 +213,9 @@ func (h *Handler) postLongURLinJSON(w http.ResponseWriter, r *http.Request) { duplicateFound = true } - log.Println("Созданный короткий идентификатор URL:", sh) + log.Println("Созданный короткий идентификатор URL:", shortURL) - response, err := json.Marshal(PostResponseBody{baseURL + sh}) + response, err := json.Marshal(PostResponseBody{baseURL + shortURL}) if err != nil { log.Println("Ошибка '", err, "' при формировании ответа:", requestBody.URL) http.Error(w, "ошибка при при формировании ответа: "+err.Error(), http.StatusInternalServerError) @@ -327,8 +327,8 @@ func (h *Handler) deleteURLs(w http.ResponseWriter, r *http.Request) { return } - for i, r := range requestBody { - requestBody[i] = strings.Replace(r, baseURL, "", -1) + for i, record := range requestBody { + requestBody[i] = strings.Replace(record, baseURL, "", -1) } log.Println("Список подлежащих удалению коротких идентификаторов URL:\n", requestBody) From a1dff0de8243f200439484d15953fdddbaba2bd2 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Mon, 6 Feb 2023 18:48:08 +0300 Subject: [PATCH 14/25] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=B7=D0=B0=D0=BC=D0=B5=D1=87?= =?UTF-8?q?=D0=B0=D0=BD=D0=B8=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/shortener/main.go | 2 +- internal/handlers/handlers.go | 19 ++-- internal/handlers/handlers_test.go | 11 +- internal/storage/db.go | 116 +++++---------------- internal/storage/file.go | 4 +- internal/storage/storage.go | 160 +++++++++++++++++++++-------- internal/storage/storage_test.go | 32 +++--- 7 files changed, 178 insertions(+), 166 deletions(-) diff --git a/cmd/shortener/main.go b/cmd/shortener/main.go index df8e391..a44dab8 100644 --- a/cmd/shortener/main.go +++ b/cmd/shortener/main.go @@ -15,7 +15,7 @@ func main() { ctx := context.Background() - str := storage.NewStorage(cfg.FileStoragePath, cfg.DatabaseDSN, ctx) + str := storage.NewStorage(ctx, cfg.FileStoragePath, cfg.DatabaseDSN) if closeFunc := str.CloseFunc(); closeFunc != nil { defer closeFunc() } diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 620f264..91e0d79 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -2,6 +2,7 @@ package handlers import ( "encoding/json" + "errors" "log" "net/http" "strings" @@ -89,21 +90,21 @@ func (h *Handler) getLongURL(w http.ResponseWriter, r *http.Request) { shortURL := strings.Trim(r.URL.Path, "/") log.Println("Идентификатор короткого URL, полученный из GET-запроса:", shortURL) - longURL, deleted, err := h.storage.FindURL(shortURL) + result, err := h.storage.FindURL(shortURL) if err != nil { log.Println("Ошибка '", err, "'. Не найден URL с указанным коротким идентификатором:", shortURL) http.Error(w, "URL с указанным коротким идентификатором не найден", http.StatusBadRequest) return } - if deleted { - log.Println("URL", longURL, "для короткого идентификатора", shortURL, "был удалён") + if result.Deleted { + log.Println("URL", result.LongURL, "для короткого идентификатора", shortURL, "был удалён") w.WriteHeader(http.StatusGone) return } - log.Println("Найден URL", longURL, "для короткого идентификатора", shortURL) - w.Header().Set("Location", longURL) + log.Println("Найден URL", result.LongURL, "для короткого идентификатора", shortURL) + w.Header().Set("Location", result.LongURL) w.WriteHeader(http.StatusTemporaryRedirect) } @@ -120,12 +121,12 @@ func (h *Handler) getLongURLsByUser(w http.ResponseWriter, r *http.Request) { response := make(shortAndLongURLs, 0) for i, shortURL := range urls { - longURL, _, err := h.storage.FindURL(shortURL) + result, err := h.storage.FindURL(shortURL) if err != nil { continue } - record := shortAndLongURL{baseURL + shortURL, longURL} + record := shortAndLongURL{baseURL + shortURL, result.LongURL} log.Println("Запись", i, "короткий URL", record.ShortURL, "длинный URL", record.LongURL) response = append(response, record) } @@ -158,7 +159,7 @@ func (h *Handler) postLongURL(w http.ResponseWriter, r *http.Request) { } shortURL, err := h.storage.AddURL(longURL, h.auth.GetUserID()) - if err != nil && !strings.Contains(err.Error(), longURL) { + if err != nil && errors.Is(err, storage.DBError{LongURL: longURL, Err: nil}) { log.Println("Ошибка '", err, "' при добавлении в БД URL:", longURL) http.Error(w, "ошибка при добавлении в БД: "+err.Error(), http.StatusInternalServerError) return @@ -203,7 +204,7 @@ func (h *Handler) postLongURLinJSON(w http.ResponseWriter, r *http.Request) { var duplicateFound bool shortURL, err := h.storage.AddURL(requestBody.URL, h.auth.GetUserID()) - if err != nil && !strings.Contains(err.Error(), requestBody.URL) { + if err != nil && errors.Is(err, storage.DBError{LongURL: requestBody.URL, Err: nil}) { log.Println("Ошибка '", err, "' при добавлении в БД URL:", requestBody.URL) http.Error(w, "ошибка при добавлении в БД: "+err.Error(), http.StatusInternalServerError) return diff --git a/internal/handlers/handlers_test.go b/internal/handlers/handlers_test.go index f91d30d..64cdbea 100644 --- a/internal/handlers/handlers_test.go +++ b/internal/handlers/handlers_test.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "errors" "io" "net/http" @@ -25,11 +26,11 @@ func (s *dummyStorage) AddURL(l, user string) (string, error) { return l, nil } -func (s *dummyStorage) FindURL(sh string) (string, bool, error) { +func (s *dummyStorage) FindURL(sh string) (storage.MemoryRecord, error) { if l, ok := s.container[sh]; ok { - return l, false, nil + return storage.MemoryRecord{l, "", false}, nil } - return "", false, errors.New("короткий URL с ID \" + string(sh) + \" не существует") + return storage.MemoryRecord{"", "", false}, errors.New("короткий URL с ID \" + string(sh) + \" не существует") } func (s *dummyStorage) GetURLsByUser(u string) []string { @@ -57,6 +58,10 @@ func (s *dummyStorage) DeleteURLs(urls []string, user string) []string { return urls } +func (s *dummyStorage) deletionQueueProcess(ctx context.Context) { + +} + func TestGzipWriter_Write(t *testing.T) { t.Skip() } diff --git a/internal/storage/db.go b/internal/storage/db.go index 0147e20..959aa89 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -4,12 +4,10 @@ import ( "context" "errors" "fmt" - "log" - "sync" - "github.com/jackc/pgerrcode" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" + "log" ) const txPreparedInsert = "shurl-insert" @@ -17,11 +15,7 @@ const txPreparedDelete = "shurl-delete" type databaseStorage struct { *memoryStorage - conn *pgx.Conn - deletionQueue chan string - deletionCancel context.CancelFunc - errors chan error - locker sync.Mutex + conn *pgx.Conn } type DBError struct { @@ -29,79 +23,8 @@ type DBError struct { Err error } -const DeletionBatchSize = 20 -const DeletionQueueSize = DeletionBatchSize * 2 - -func (s *databaseStorage) DeleteURLs(shortURLs []string, user string) (deleted []string) { - deleted = make([]string, 0) - - go func() { - deleted = s.memoryStorage.DeleteURLs(shortURLs, user) - - for _, sh := range deleted { - s.deletionQueue <- sh - } - }() - - return deleted -} - -func (s *databaseStorage) deletionQueueProcess(ctx context.Context) chan error { - errorChan := make(chan error) - - go func(s *databaseStorage, ctx context.Context) { - deletionBatch := make([]string, DeletionBatchSize) - for { - select { - case sh, ok := <-s.deletionQueue: - if !ok { - return - } - - deletionBatch = append(deletionBatch, sh) - - if len(deletionBatch) >= DeletionBatchSize { - err := s.delete(ctx, deletionBatch) - if err != nil { - errorChan <- err - } - deletionBatch = deletionBatch[:0] - } - - case <-ctx.Done(): - return - default: - if len(deletionBatch) == 0 { - continue - } - - err := s.delete(ctx, deletionBatch) - if err != nil { - errorChan <- err - } - deletionBatch = deletionBatch[:0] - } - } - }(s, ctx) - - return errorChan -} - -func (s *databaseStorage) errorProcess(ctx context.Context) { - go func() { - for { - select { - case err, ok := <-s.errors: - if !ok { - return - } - - log.Println(err) - case <-ctx.Done(): - return - } - } - }() +func (s *databaseStorage) deletionQueueProcess(ctx context.Context) { + go deletionQueueProcess(ctx, s, s.memoryStorage.deletionQueue) } func (s *databaseStorage) delete(ctx context.Context, deletionBatch []string) error { @@ -131,11 +54,12 @@ func (s *databaseStorage) delete(ctx context.Context, deletionBatch []string) er if err != nil { return err } - return nil + + return s.memoryStorage.delete(ctx, deletionBatch) } func newDBStorage(ctx context.Context, m *memoryStorage, database string) *databaseStorage { - storage := &databaseStorage{memoryStorage: m, conn: nil, deletionQueue: nil} + storage := &databaseStorage{memoryStorage: m, conn: nil} var err error storage.conn, err = pgx.Connect(ctx, database) @@ -149,13 +73,6 @@ func newDBStorage(ctx context.Context, m *memoryStorage, database string) *datab log.Fatal(err) } - storage.deletionQueue = make(chan string, DeletionQueueSize) - - var deletionCtx context.Context - deletionCtx, storage.deletionCancel = context.WithCancel(ctx) - - storage.errors = storage.deletionQueueProcess(deletionCtx) - return storage } @@ -181,7 +98,7 @@ func (s *databaseStorage) init(ctx context.Context) error { log.Println("Ошибка чтения из БД:", err) } - s.memoryStorage.container[sh] = memoryRecord{longURL: l, deleted: d, user: u} + s.memoryStorage.container[sh] = MemoryRecord{LongURL: l, Deleted: d, User: u} s.memoryStorage.usersURLs[u] = append(s.memoryStorage.usersURLs[u], sh) } @@ -194,10 +111,23 @@ func (s *databaseStorage) init(ctx context.Context) error { return nil } -func (e *DBError) Error() string { +func (e DBError) Error() string { return fmt.Sprintf("Найден дубликат для полного URL: %v. Ошибка добавления в БД: %v", e.LongURL, e.Err) } +func (e DBError) Is(target error) bool { + err, ok := target.(DBError) + if !ok { + return false + } + + if err.LongURL != e.LongURL { + return false + } + + return true +} + func NewStorageDBError(longURL string, err error) error { return &DBError{ LongURL: longURL, @@ -289,7 +219,7 @@ func (s *databaseStorage) CloseFunc() func() { return func() { s.deletionCancel() close(s.deletionQueue) - close(s.errors) + //close(s.errors) if s.conn == nil { return diff --git a/internal/storage/file.go b/internal/storage/file.go index 6ce6860..ec4fd49 100644 --- a/internal/storage/file.go +++ b/internal/storage/file.go @@ -69,7 +69,7 @@ func (s *fileStorage) loadFromFile() error { if r.ShortURL == "" || r.LongURL == "" { continue } - s.container[r.ShortURL] = memoryRecord{longURL: r.LongURL, deleted: r.Deleted, user: r.UserID} + s.container[r.ShortURL] = MemoryRecord{LongURL: r.LongURL, Deleted: r.Deleted, User: r.UserID} if r.UserID == "" { continue @@ -130,7 +130,7 @@ func (s *fileStorage) DeleteURLs(shortURLs []string, user string) (deleted []str deleted = s.memoryStorage.DeleteURLs(shortURLs, user) for _, sh := range deleted { - err := s.saveToFile(&Record{ShortURL: sh, LongURL: s.container[sh].longURL, Deleted: true, UserID: user}) + err := s.saveToFile(&Record{ShortURL: sh, LongURL: s.container[sh].LongURL, Deleted: true, UserID: user}) if err != nil { log.Println("Ошибка при записи удалённой ссылки с id", sh, "в файл") } diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 8af4690..9359f7b 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -11,6 +11,9 @@ import ( "time" ) +const DeletionBatchSize = 20 +const DeletionQueueSize = DeletionBatchSize * 2 + type RecordURL struct { ID string URL string @@ -21,39 +24,62 @@ type BatchURLs = []RecordURL type Storager interface { AddURL(string, string) (string, error) AddURLs(BatchURLs, string) (BatchURLs, error) - FindURL(string) (string, bool, error) + FindURL(string) (MemoryRecord, error) GetURLsByUser(string) []string DeleteURLs([]string, string) []string CloseFunc() func() Ping() error } -type memoryRecord struct { - longURL string - user string - deleted bool +type deleter interface { + deletionQueueProcess(context.Context) + delete(context.Context, []string) error } -type memoryStorage struct { - container map[string]memoryRecord - usersURLs map[string][]string - locker sync.RWMutex +type MemoryRecord struct { + LongURL string + User string + Deleted bool } -func NewStorage(filePath string, database string, ctx context.Context) Storager { - if database != "" { - return newDBStorage(ctx, newMemoryStorage(), database) - } +type memoryStorage struct { + container map[string]MemoryRecord + usersURLs map[string][]string + locker sync.RWMutex + deletionQueue chan string + deletionCancel context.CancelFunc +} - if filePath != "" { - return newFileStorage(newMemoryStorage(), filePath) +func NewStorage(ctx context.Context, filePath string, database string) Storager { + var storage Storager + + deletionContext, deletionCancel := context.WithCancel(ctx) + + switch { + case database != "": + dStorage := newDBStorage(ctx, newMemoryStorage(), database) + dStorage.deletionCancel = deletionCancel + dStorage.deletionQueueProcess(deletionContext) + storage = dStorage + + case filePath != "": + fStorage := newFileStorage(newMemoryStorage(), filePath) + fStorage.deletionCancel = deletionCancel + fStorage.deletionQueueProcess(deletionContext) + storage = fStorage + + default: + mStorage := newMemoryStorage() + mStorage.deletionCancel = deletionCancel + mStorage.deletionQueueProcess(deletionContext) + storage = mStorage } - return newMemoryStorage() + return storage } func newMemoryStorage() *memoryStorage { - return &memoryStorage{map[string]memoryRecord{}, map[string][]string{}, sync.RWMutex{}} + return &memoryStorage{map[string]MemoryRecord{}, map[string][]string{}, sync.RWMutex{}, make(chan string, DeletionQueueSize), nil} } func generateShortURL() (string, error) { @@ -85,7 +111,7 @@ func (s *memoryStorage) AddURL(l, user string) (string, error) { return "", errors.New("короткий URL с ID " + string(sh) + " уже существует") } - s.container[sh] = memoryRecord{longURL: l, deleted: false, user: user} + s.container[sh] = MemoryRecord{LongURL: l, Deleted: false, User: user} s.usersURLs[user] = append(s.usersURLs[user], sh) return sh, nil } @@ -107,20 +133,16 @@ func (s *memoryStorage) AddURLs(longURLs BatchURLs, user string) (BatchURLs, err return result, nil } -func (s *memoryStorage) FindURL(sh string) (string, bool, error) { +func (s *memoryStorage) FindURL(sh string) (MemoryRecord, error) { s.locker.RLock() defer s.locker.RUnlock() - r, ok := s.container[sh] + result, ok := s.container[sh] if !ok { - return "", false, errors.New("короткий URL с ID \" + string(sh) + \" не существует") + return MemoryRecord{"", "", false}, errors.New("короткий URL с ID \" + string(sh) + \" не существует") } - if r.deleted { - return "", r.deleted, nil - } - - return r.longURL, r.deleted, nil + return result, nil } func (s *memoryStorage) GetURLsByUser(u string) []string { @@ -130,33 +152,85 @@ func (s *memoryStorage) GetURLsByUser(u string) []string { return s.usersURLs[u] } +func (s *memoryStorage) CloseFunc() func() { + return nil +} + +func (s *memoryStorage) Ping() error { + return errors.New("БД не была подключена, используется хранилище в памяти") +} + func (s *memoryStorage) DeleteURLs(shortURLs []string, user string) (deleted []string) { - deleted = make([]string, 0) + go func() { + s.locker.RLock() + defer s.locker.RUnlock() - s.locker.Lock() - defer s.locker.Unlock() + for _, shortURL := range shortURLs { + mr, ok := s.container[shortURL] + if !ok { + continue + } - for _, sh := range shortURLs { - mr, ok := s.container[sh] - if !ok { - continue - } + if mr.User != user || mr.Deleted { + continue + } - if mr.user != user { - continue + s.deletionQueue <- shortURL } - - s.container[sh] = memoryRecord{longURL: mr.longURL, user: mr.user, deleted: true} - deleted = append(deleted, sh) - } + }() return deleted } -func (s *memoryStorage) CloseFunc() func() { +func (s *memoryStorage) delete(ctx context.Context, deletionBatch []string) error { + s.locker.Lock() + defer s.locker.Unlock() + + for _, shortURL := range deletionBatch { + mr := s.container[shortURL] + mr.Deleted = true + s.container[shortURL] = mr + } + return nil } -func (s *memoryStorage) Ping() error { - return errors.New("БД не была подключена, используется хранилище в памяти") +func (s *memoryStorage) deletionQueueProcess(ctx context.Context) { + go deletionQueueProcess(ctx, s, s.deletionQueue) +} + +func deletionQueueProcess(ctx context.Context, d deleter, deletionQueue <-chan string) { + deletionBatch := make([]string, DeletionBatchSize) + + for { + select { + case sh, ok := <-deletionQueue: + if !ok { + return + } + + deletionBatch = append(deletionBatch, sh) + + if len(deletionBatch) >= DeletionBatchSize { + err := d.delete(ctx, deletionBatch) + if err != nil { + log.Println(err) + } + deletionBatch = deletionBatch[:0] + } + + case <-ctx.Done(): + return + default: + if len(deletionBatch) == 0 { + continue + } + + err := d.delete(ctx, deletionBatch) + if err != nil { + log.Println(err) + } + deletionBatch = deletionBatch[:0] + } + } } diff --git a/internal/storage/storage_test.go b/internal/storage/storage_test.go index eeb5f22..29700c8 100644 --- a/internal/storage/storage_test.go +++ b/internal/storage/storage_test.go @@ -75,7 +75,7 @@ func Test_memoryStorage_AddURL(t *testing.T) { }{ { "Успешное добавление 1 элемента", - &memoryStorage{map[string]memoryRecord{}, map[string][]string{}, sync.RWMutex{}}, + &memoryStorage{map[string]MemoryRecord{}, map[string][]string{}, sync.RWMutex{}, nil, nil}, "http://ya.ru", "1111122222", 1, @@ -83,7 +83,7 @@ func Test_memoryStorage_AddURL(t *testing.T) { }, { "Успешное добавление дублирующих элементов", - &memoryStorage{map[string]memoryRecord{}, map[string][]string{}, sync.RWMutex{}}, + &memoryStorage{map[string]MemoryRecord{}, map[string][]string{}, sync.RWMutex{}, nil, nil}, "http://ya.ru", "3333344444", 3, @@ -111,32 +111,32 @@ func Test_memoryStorage_FindURL(t *testing.T) { }{ { "Неуспешная попытка поиска в пустом хранилище", - &memoryStorage{map[string]memoryRecord{}, map[string][]string{}, sync.RWMutex{}}, + &memoryStorage{map[string]MemoryRecord{}, map[string][]string{}, sync.RWMutex{}, nil, nil}, "dummy", "", false, }, { "Успешная попытка поиска в списке из 1 элемента", - &memoryStorage{map[string]memoryRecord{"dummy": {"http://ya.ru", "", false}}, map[string][]string{}, sync.RWMutex{}}, + &memoryStorage{map[string]MemoryRecord{"dummy": {"http://ya.ru", "", false}}, map[string][]string{}, sync.RWMutex{}, nil, nil}, "dummy", "http://ya.ru", true, }, { "Успешная попытка поиска в списке из 3 элементов", - &memoryStorage{map[string]memoryRecord{ + &memoryStorage{map[string]MemoryRecord{ "dummy": {"http://ya.ru", "", false}, "dummy1": {"http://mail.ru", "", false}, "dummy2": {"http://google.ru", "", false}, - }, map[string][]string{}, sync.RWMutex{}}, + }, map[string][]string{}, sync.RWMutex{}, nil, nil}, "dummy1", "http://mail.ru", true, }, { "Неуспешная попытка поиска в непустом списке", - &memoryStorage{map[string]memoryRecord{"dummy": {"http://ya.ru", "", false}}, map[string][]string{}, sync.RWMutex{}}, + &memoryStorage{map[string]MemoryRecord{"dummy": {"http://ya.ru", "", false}}, map[string][]string{}, sync.RWMutex{}, nil, nil}, "dummy1", "", false, @@ -144,9 +144,9 @@ func Test_memoryStorage_FindURL(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - l, _, err := tt.s.FindURL(tt.URL) + result, err := tt.s.FindURL(tt.URL) assert.Equal(t, tt.OK, err == nil) - assert.Equal(t, tt.wantURL, l) + assert.Equal(t, tt.wantURL, result.LongURL) }) } } @@ -161,27 +161,29 @@ func Test_fileStorage_AddURL(t *testing.T) { }{ { "Неуспешная попытка поиска в пустом хранилище", - fileStorage{&memoryStorage{map[string]memoryRecord{}, map[string][]string{}, sync.RWMutex{}}, nil, nil, nil}, + fileStorage{&memoryStorage{map[string]MemoryRecord{}, map[string][]string{}, sync.RWMutex{}, nil, nil}, nil, nil, nil}, "dummy", "", false, }, { "Успешная попытка поиска в списке из 1 элемента", - fileStorage{&memoryStorage{map[string]memoryRecord{"dummy": {"http://ya.ru", "", false}}, map[string][]string{}, sync.RWMutex{}}, nil, nil, nil}, + fileStorage{&memoryStorage{map[string]MemoryRecord{"dummy": {"http://ya.ru", "", false}}, map[string][]string{}, sync.RWMutex{}, nil, nil}, nil, nil, nil}, "dummy", "http://ya.ru", true, }, { "Успешная попытка поиска в списке из 3 элементов", - fileStorage{&memoryStorage{map[string]memoryRecord{ + fileStorage{&memoryStorage{map[string]MemoryRecord{ "dummy": {"http://ya.ru", "", false}, "dummy1": {"http://mail.ru", "", false}, "dummy2": {"http://google.ru", "", false}, }, map[string][]string{}, sync.RWMutex{}, + nil, + nil, }, nil, nil, nil}, "dummy1", @@ -190,7 +192,7 @@ func Test_fileStorage_AddURL(t *testing.T) { }, { "Неуспешная попытка поиска в непустом списке", - fileStorage{&memoryStorage{map[string]memoryRecord{"dummy": {"http://ya.ru", "", false}}, map[string][]string{}, sync.RWMutex{}}, nil, nil, nil}, + fileStorage{&memoryStorage{map[string]MemoryRecord{"dummy": {"http://ya.ru", "", false}}, map[string][]string{}, sync.RWMutex{}, nil, nil}, nil, nil, nil}, "dummy1", "", false, @@ -198,9 +200,9 @@ func Test_fileStorage_AddURL(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - l, _, err := tt.s.FindURL(tt.URL) + result, err := tt.s.FindURL(tt.URL) assert.Equal(t, tt.OK, err == nil) - assert.Equal(t, tt.wantURL, l) + assert.Equal(t, tt.wantURL, result.LongURL) }) } } From 00ea34ce95f111201e338c0a45044b67be9c947b Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Mon, 6 Feb 2023 18:53:24 +0300 Subject: [PATCH 15/25] Update handlers_test.go --- internal/handlers/handlers_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/handlers/handlers_test.go b/internal/handlers/handlers_test.go index 64cdbea..3a5a41f 100644 --- a/internal/handlers/handlers_test.go +++ b/internal/handlers/handlers_test.go @@ -28,9 +28,9 @@ func (s *dummyStorage) AddURL(l, user string) (string, error) { func (s *dummyStorage) FindURL(sh string) (storage.MemoryRecord, error) { if l, ok := s.container[sh]; ok { - return storage.MemoryRecord{l, "", false}, nil + return storage.MemoryRecord{LongURL: l, User: "", Deleted: false}, nil } - return storage.MemoryRecord{"", "", false}, errors.New("короткий URL с ID \" + string(sh) + \" не существует") + return storage.MemoryRecord{LongURL: "", User: "", Deleted: false}, errors.New("короткий URL с ID \" + string(sh) + \" не существует") } func (s *dummyStorage) GetURLsByUser(u string) []string { From dae9bdcb8279f6fae3a9f1fdf60a698573a9c372 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Mon, 6 Feb 2023 21:54:55 +0300 Subject: [PATCH 16/25] Update shortenertest.yml --- .github/workflows/shortenertest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/shortenertest.yml b/.github/workflows/shortenertest.yml index 71edff5..d9aa356 100644 --- a/.github/workflows/shortenertest.yml +++ b/.github/workflows/shortenertest.yml @@ -54,7 +54,7 @@ jobs: - name: Build server binary run: | cd cmd/shortener - go build -o shortener + go build buildvcs=false -o shortener - name: "Code increment #1" if: | From eae6d4d6f516e9892b3216342bcd54bd8c1707a5 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Mon, 6 Feb 2023 21:56:35 +0300 Subject: [PATCH 17/25] Update shortenertest.yml --- .github/workflows/shortenertest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/shortenertest.yml b/.github/workflows/shortenertest.yml index d9aa356..7b22c2b 100644 --- a/.github/workflows/shortenertest.yml +++ b/.github/workflows/shortenertest.yml @@ -54,7 +54,7 @@ jobs: - name: Build server binary run: | cd cmd/shortener - go build buildvcs=false -o shortener + go build -buildvcs=false -o shortener - name: "Code increment #1" if: | From 214a19167b44f7d6c149d11b804c6a93ac8c05b7 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Mon, 6 Feb 2023 22:01:25 +0300 Subject: [PATCH 18/25] Update main.go --- cmd/shortener/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/shortener/main.go b/cmd/shortener/main.go index a44dab8..1a23664 100644 --- a/cmd/shortener/main.go +++ b/cmd/shortener/main.go @@ -19,7 +19,7 @@ func main() { if closeFunc := str.CloseFunc(); closeFunc != nil { defer closeFunc() } - + h := handlers.NewHandler(str, cfg.BaseURL) srv := server.NewServer(cfg.ServerAddress, h) From 7cd014fcad7ebc1e908957ea86d23089169081f1 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Mon, 6 Feb 2023 22:06:19 +0300 Subject: [PATCH 19/25] Update main.go --- cmd/shortener/main.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cmd/shortener/main.go b/cmd/shortener/main.go index 1a23664..31f4b0f 100644 --- a/cmd/shortener/main.go +++ b/cmd/shortener/main.go @@ -12,14 +12,12 @@ import ( func main() { cfg := config.NewConfiguration() - ctx := context.Background() - str := storage.NewStorage(ctx, cfg.FileStoragePath, cfg.DatabaseDSN) if closeFunc := str.CloseFunc(); closeFunc != nil { defer closeFunc() } - + h := handlers.NewHandler(str, cfg.BaseURL) srv := server.NewServer(cfg.ServerAddress, h) From 7cc5868776354603d8a0b7dac12d833ecd701f0d Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Mon, 6 Feb 2023 22:10:59 +0300 Subject: [PATCH 20/25] . --- cmd/shortener/main.go | 1 + internal/handlers/handlers.go | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/shortener/main.go b/cmd/shortener/main.go index 31f4b0f..cc3be1b 100644 --- a/cmd/shortener/main.go +++ b/cmd/shortener/main.go @@ -13,6 +13,7 @@ import ( func main() { cfg := config.NewConfiguration() ctx := context.Background() + str := storage.NewStorage(ctx, cfg.FileStoragePath, cfg.DatabaseDSN) if closeFunc := str.CloseFunc(); closeFunc != nil { defer closeFunc() diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 91e0d79..2aa2e6f 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -243,7 +243,6 @@ func (h *Handler) ping(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusInternalServerError) return } - w.WriteHeader(http.StatusOK) } From 579ea213886a755775b914eec188612fc6ea2c94 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Tue, 27 Jun 2023 18:53:13 +0300 Subject: [PATCH 21/25] polish --- .DS_Store | Bin 6148 -> 8196 bytes internal/auth/auth.go | 7 ++++--- internal/handlers/handlers.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.DS_Store b/.DS_Store index 69cdf7bd32373872264fd24f67cac91a862415d8..607774bd55b579eb39943154ccafc305f1fbe0a7 100644 GIT binary patch literal 8196 zcmeHM&ubGw6n>MWO=1y3Q7nkCqDa9|Qx8$`lE(Dlp_fMVV5QA=W3qO$6Y`@0Yav(l z;zjhL7ykgkgC0BxQo)0Vo&^5~@!lS)Z+@hi+0FJMNYxqG`8M;uH~Zc<)0v$u0FYwW znFW{vfD~Q~GZR>iD9p~TP>IQwvxo%!;krjxZI9acI}>7?2BUycz$jo8FbWt24ub-? zXS0%u*!P9i)J6fLz<;R#TOTaE7#1Y<1@ff>D}Dq(OkguFc+X$`5iQn$SdiEkh%H!% za77fZL>@7SJUI~Ij`o7&_XP@fApFVLk9;zbCln$N9>n5wAO(S@HVPO8!V0jndje*l z1}=2;^E=r=+ssd1xZ^(0tId{IX`{(oll2E`9}QY6$!`Fu?ddP4w?1#p>2~Kpd$X?e zef|l1L~;-x)S(Nu9%ZbC3^mFCTZGiz+_rRFtto$PbhgE2RGa^cLOeD+hwVQHoytA90+wSCYUr8*U7#mMwnMmF#*J-cZsdT#S zVx@JP{klP^XAkZx_D#3iTh1K2M*U93^{XxJ;5t>LEZuT_hxUrJ>pN}kTNZ{imgUTe zjg3qBtTmg31Oi-x#=%4Ehj$`++iqIO4& z;Kd5#^#yVltoU&pksrqq_y1vt_ZCu_PeEc|Ahuxn;~xT)*C1_Y1hfA`Dr#?6Us|I$ MX8-3C`Y)+|0k``AkN^Mx delta 127 zcmZp1XfcprU|?W$DortDU=RQ@Ie-{Mvv5r;6q~50$jGxXU^g=(&tx6}?a6lq_$TiZ zke{3@7&qBl=-1}0!oiG_7l=AeEZer2or6P=8K@Bm1h|2OE69wEh2NPc^UHXGOk`k! Om=3a#VRJms9A*Hm85xQI diff --git a/internal/auth/auth.go b/internal/auth/auth.go index d3cabf7..904775c 100644 --- a/internal/auth/auth.go +++ b/internal/auth/auth.go @@ -116,10 +116,11 @@ func (a *authentication) Authenticate(next http.Handler) http.Handler { return } + a.userID = "" + a.cookieFull = "" + cookie, err := r.Cookie(cookieAuthentication) if err != nil { - a.cookieFull = "" - a.userID = "" log.Println("Cookie '" + cookieAuthentication + "' не переданы") } @@ -128,7 +129,7 @@ func (a *authentication) Authenticate(next http.Handler) http.Handler { err = a.authExisting(cookie.Value) } if err != nil { - log.Println("Ошибка при чтении cookie:", err) + log.Println("Ошибка при аутентификации пользователя через cookie 'authentication':", err) } err = nil diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 2aa2e6f..9a5609a 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -228,7 +228,7 @@ func (h *Handler) postLongURLinJSON(w http.ResponseWriter, r *http.Request) { if duplicateFound { w.WriteHeader(http.StatusConflict) } else { - w.WriteHeader(http.StatusCreated) + w.WriteHeader(http.StatusOK) } _, err = w.Write(response) if err != nil { From 3f00a45a80e416e613ed6f6ead67a3cb901472f9 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Tue, 27 Jun 2023 19:11:10 +0300 Subject: [PATCH 22/25] Update handlers.go --- internal/handlers/handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 9a5609a..2aa2e6f 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -228,7 +228,7 @@ func (h *Handler) postLongURLinJSON(w http.ResponseWriter, r *http.Request) { if duplicateFound { w.WriteHeader(http.StatusConflict) } else { - w.WriteHeader(http.StatusOK) + w.WriteHeader(http.StatusCreated) } _, err = w.Write(response) if err != nil { From 96fc8ea47bb7106ad2a2f7f695acc68b81a82e6c Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Tue, 27 Jun 2023 19:36:00 +0300 Subject: [PATCH 23/25] undo locks --- internal/storage/db.go | 6 ++++-- internal/storage/storage.go | 5 ++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/internal/storage/db.go b/internal/storage/db.go index 959aa89..22376ca 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -136,13 +136,15 @@ func NewStorageDBError(longURL string, err error) error { } func (s *databaseStorage) AddURL(l, user string) (string, error) { + log.Println(l, user) + sh, err := s.memoryStorage.AddURL(l, user) if err != nil { return "", err } - s.locker.Lock() - defer s.locker.Unlock() + //s.locker.Lock() + //defer s.locker.Unlock() ctx := context.Background() var pgErr *pgconn.PgError diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 9359f7b..4218309 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -99,9 +99,6 @@ func generateShortURL() (string, error) { } func (s *memoryStorage) AddURL(l, user string) (string, error) { - s.locker.Lock() - defer s.locker.Unlock() - sh, err := generateShortURL() if err != nil { return "", err @@ -111,6 +108,8 @@ func (s *memoryStorage) AddURL(l, user string) (string, error) { return "", errors.New("короткий URL с ID " + string(sh) + " уже существует") } + //s.locker.Lock() + //defer s.locker.Unlock() s.container[sh] = MemoryRecord{LongURL: l, Deleted: false, User: user} s.usersURLs[user] = append(s.usersURLs[user], sh) return sh, nil From ffa2e7e204ea4a121857499f948a14ed9e0dd739 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Tue, 27 Jun 2023 20:12:11 +0300 Subject: [PATCH 24/25] mutex --- internal/storage/db.go | 5 ++--- internal/storage/storage.go | 16 +++++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/internal/storage/db.go b/internal/storage/db.go index 22376ca..154b7be 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -136,15 +136,14 @@ func NewStorageDBError(longURL string, err error) error { } func (s *databaseStorage) AddURL(l, user string) (string, error) { - log.Println(l, user) sh, err := s.memoryStorage.AddURL(l, user) if err != nil { return "", err } - //s.locker.Lock() - //defer s.locker.Unlock() + s.locker.Lock() + defer s.locker.Unlock() ctx := context.Background() var pgErr *pgconn.PgError diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 4218309..6feed5b 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -79,7 +79,12 @@ func NewStorage(ctx context.Context, filePath string, database string) Storager } func newMemoryStorage() *memoryStorage { - return &memoryStorage{map[string]MemoryRecord{}, map[string][]string{}, sync.RWMutex{}, make(chan string, DeletionQueueSize), nil} + return &memoryStorage{ + container: map[string]MemoryRecord{}, + usersURLs: map[string][]string{}, + deletionQueue: make(chan string, DeletionQueueSize), + deletionCancel: nil, + } } func generateShortURL() (string, error) { @@ -99,6 +104,9 @@ func generateShortURL() (string, error) { } func (s *memoryStorage) AddURL(l, user string) (string, error) { + s.locker.Lock() + defer s.locker.Unlock() + sh, err := generateShortURL() if err != nil { return "", err @@ -108,8 +116,6 @@ func (s *memoryStorage) AddURL(l, user string) (string, error) { return "", errors.New("короткий URL с ID " + string(sh) + " уже существует") } - //s.locker.Lock() - //defer s.locker.Unlock() s.container[sh] = MemoryRecord{LongURL: l, Deleted: false, User: user} s.usersURLs[user] = append(s.usersURLs[user], sh) return sh, nil @@ -182,8 +188,8 @@ func (s *memoryStorage) DeleteURLs(shortURLs []string, user string) (deleted []s } func (s *memoryStorage) delete(ctx context.Context, deletionBatch []string) error { - s.locker.Lock() - defer s.locker.Unlock() + //s.locker.Lock() + //defer s.locker.Unlock() for _, shortURL := range deletionBatch { mr := s.container[shortURL] From e7e21820a2cf10ecde9baa5fb82a92576f60bc68 Mon Sep 17 00:00:00 2001 From: StainlessSteelSnake Date: Wed, 28 Jun 2023 10:21:08 +0300 Subject: [PATCH 25/25] DUPLICATE --- internal/handlers/handlers.go | 10 ++++++++-- internal/storage/db.go | 18 ++++++++++-------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.go index 2aa2e6f..47f2d77 100644 --- a/internal/handlers/handlers.go +++ b/internal/handlers/handlers.go @@ -159,15 +159,21 @@ func (h *Handler) postLongURL(w http.ResponseWriter, r *http.Request) { } shortURL, err := h.storage.AddURL(longURL, h.auth.GetUserID()) - if err != nil && errors.Is(err, storage.DBError{LongURL: longURL, Err: nil}) { + if err != nil && errors.Is(err, storage.DBError{LongURL: longURL, Duplicate: false, Err: nil}) { log.Println("Ошибка '", err, "' при добавлении в БД URL:", longURL) http.Error(w, "ошибка при добавлении в БД: "+err.Error(), http.StatusInternalServerError) return } - if err != nil { + if err != nil && errors.Is(err, storage.DBError{LongURL: longURL, Duplicate: true, Err: nil}) { log.Println("Найденный короткий идентификатор URL:", shortURL) w.WriteHeader(http.StatusConflict) + err = nil + } + + if err != nil { + log.Println("Ошибка '", err, "' при добавлении в БД URL:", longURL) + w.WriteHeader(http.StatusInternalServerError) } else { log.Println("Созданный короткий идентификатор URL:", shortURL) w.WriteHeader(http.StatusCreated) diff --git a/internal/storage/db.go b/internal/storage/db.go index 154b7be..69529f7 100644 --- a/internal/storage/db.go +++ b/internal/storage/db.go @@ -19,8 +19,9 @@ type databaseStorage struct { } type DBError struct { - LongURL string - Err error + LongURL string + Duplicate bool + Err error } func (s *databaseStorage) deletionQueueProcess(ctx context.Context) { @@ -121,17 +122,18 @@ func (e DBError) Is(target error) bool { return false } - if err.LongURL != e.LongURL { + if err.LongURL != e.LongURL || err.Duplicate != e.Duplicate { return false } return true } -func NewStorageDBError(longURL string, err error) error { +func NewStorageDBError(longURL string, duplicate bool, err error) error { return &DBError{ - LongURL: longURL, - Err: err, + LongURL: longURL, + Duplicate: duplicate, + Err: err, } } @@ -159,12 +161,12 @@ func (s *databaseStorage) AddURL(l, user string) (string, error) { if err != nil { log.Println("Ошибка операции с БД, код:", pgErr.Code, ", сообщение:", pgErr.Error()) - duplicateErr := NewStorageDBError(l, err) + duplicateErr := NewStorageDBError(l, true, err) r := s.conn.QueryRow(ctx, querySelectByLongURL, l) err = r.Scan(&sh) if err != nil { - return "", NewStorageDBError(l, err) + return "", NewStorageDBError(l, false, err) } log.Println("Найдена ранее сохранённая запись")