Add a flag to the reindex command to force a full reindex

That can be helpful when the search service configuration has changed,
e.g. by enabling TIKA. Previously files that had already been indexed
were not indexed again and thus were no part of the fulltext index.

Fixes opencloud-eu#2285
Fixes opencloud-eu#2578

Author: aduffeck

protogen/gen/opencloud/services/search/v0/search.pb.go

@@ -329,6 +329,9 @@
         },
         "userId": {
           "type": "string"
+        },
+        "forceReindex": {
+          "type": "boolean"
         }
       }
     },

protogen/gen/opencloud/services/search/v0/search.swagger.json

@@ -104,6 +104,7 @@ message SearchIndexResponse {
 message IndexSpaceRequest {
   string space_id = 1;
   string user_id = 2;
+  bool force_reindex = 3;
 }
 
 message IndexSpaceResponse {

protogen/proto/opencloud/services/search/v0/search.proto

@@ -118,6 +118,13 @@ It can also be used to re-index all spaces:
 opencloud search index --all-spaces
 ```
 
+Please note that a reindex only picks up new files. Files that have already been indexed are not indexed again, even if the configuration or the whole extractor has been changed. To force a full reindex you need to use the `force-reindex` flag:
+
+
+```shell
+opencloud search index --all-spaces --force-reindex
+```
+
 ## Metrics
 
 The search service exposes the following prometheus metrics at `<debug_endpoint>/metrics` (as configured using the `SEARCH_DEBUG_ADDR` env var):

services/antivirus/pkg/scanners/mocks/scanner.go

@@ -29,6 +29,7 @@ func Index(cfg *config.Config) *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			allSpacesFlag, _ := cmd.Flags().GetBool("all-spaces")
 			spaceFlag, _ := cmd.Flags().GetString("space")
+			forceReindexFlag, _ := cmd.Flags().GetBool("force-reindex")
 			if spaceFlag == "" && !allSpacesFlag {
 				return errors.New("either --space or --all-spaces is required")
 			}
@@ -48,7 +49,8 @@ func Index(cfg *config.Config) *cobra.Command {
 
 			c := searchsvc.NewSearchProviderService("eu.opencloud.api.search", grpcClient)
 			_, err = c.IndexSpace(context.Background(), &searchsvc.IndexSpaceRequest{
-				SpaceId: spaceFlag,
+				SpaceId:      spaceFlag,
+				ForceReindex: forceReindexFlag,
 			}, func(opts *client.CallOptions) { opts.RequestTimeout = 10 * time.Minute })
 			if err != nil {
 				fmt.Println("failed to index space: " + err.Error())
@@ -68,6 +70,11 @@ func Index(cfg *config.Config) *cobra.Command {
 		false,
 		"index all spaces instead. This or --space is required.",
 	)
+	indexCmd.Flags().Bool(
+		"force-rescan",
+		false,
+		"force a rescan of all files, even if they are already indexed. This will make the indexing process much slower, but ensures that the index is up-to-date using the current search service configuration.",
+	)
 
 	return indexCmd
 }

services/search/README.md

@@ -46,7 +46,7 @@ const (
 type Searcher interface {
 	Search(ctx context.Context, req *searchsvc.SearchRequest) (*searchsvc.SearchResponse, error)
 
-	IndexSpace(rID *provider.StorageSpaceId) error
+	IndexSpace(rID *provider.StorageSpaceId, forceRescan bool) error
 	PurgeDeleted(spaceID *provider.StorageSpaceId) error
 
 	TrashItem(rID *provider.ResourceId)
@@ -443,7 +443,7 @@ func (s *Service) searchIndex(ctx context.Context, req *searchsvc.SearchRequest,
 }
 
 // IndexSpace (re)indexes all resources of a given space.
-func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId) error {
+func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId, forceRescan bool) error {
 	ownerCtx, err := getAuthContext(s.serviceAccountID, s.gatewaySelector, s.serviceAccountSecret, s.logger)
 	if err != nil {
 		return err
@@ -501,6 +501,11 @@ func (s *Service) IndexSpace(spaceID *provider.StorageSpaceId) error {
 		}
 		s.logger.Debug().Str("path", ref.Path).Msg("Walking tree")
 
+		if forceRescan {
+			s.doUpsertItem(ref, batch)
+			return nil
+		}
+
 		searchRes, err := s.engine.Search(ownerCtx, &searchsvc.SearchIndexRequest{
 			Query: "id:" + storagespace.FormatResourceID(info.Id) + ` mtime>=` + utils.TSToTime(info.Mtime).Format(time.RFC3339Nano),
 		})

services/search/pkg/command/index.go

@@ -74,7 +74,7 @@ func New(ctx context.Context, stream raw.Stream, logger log.Logger, tp trace.Tra
 	}
 
 	svc.indexSpaceDebouncer = NewSpaceDebouncer(time.Duration(debounceDuration)*time.Millisecond, 30*time.Second, func(id *provider.StorageSpaceId) {
-		if err := svc.index.IndexSpace(id); err != nil {
+		if err := svc.index.IndexSpace(id, false); err != nil {
 			svc.log.Error().Err(err).Interface("spaceID", id).Msg("error while indexing a space")
 		}
 	}, svc.log)

services/search/pkg/search/mocks/searcher.go

@@ -121,7 +121,7 @@ func (s Service) Search(ctx context.Context, in *searchsvc.SearchRequest, out *s
 // IndexSpace (re)indexes all resources of a given space.
 func (s Service) IndexSpace(_ context.Context, in *searchsvc.IndexSpaceRequest, _ *searchsvc.IndexSpaceResponse) error {
 	if in.GetSpaceId() != "" {
-		return s.searcher.IndexSpace(&provider.StorageSpaceId{OpaqueId: in.GetSpaceId()})
+		return s.searcher.IndexSpace(&provider.StorageSpaceId{OpaqueId: in.GetSpaceId()}, in.GetForceReindex())
 	}
 
 	// index all spaces instead
@@ -145,7 +145,7 @@ func (s Service) IndexSpace(_ context.Context, in *searchsvc.IndexSpaceRequest,
 	}
 
 	for _, space := range resp.GetStorageSpaces() {
-		if err := s.searcher.IndexSpace(space.GetId()); err != nil {
+		if err := s.searcher.IndexSpace(space.GetId(), in.GetForceReindex()); err != nil {
 			return err
 		}
 	}