Merge branch '8484-sanity-check-collection-count' closes #8484

diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index 556a5c047120923aa557a5fe38eab002313aaad3..b1154a8399e478d9fd2147fbb5fdb6cc8d46e016 100644 (file)
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -244,6 +244,7 @@ real_log_collection:
   uuid: zzzzz-4zz18-op4e2lbej01tcvu
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
   created_at: 2014-09-01 12:00:00
+  modified_at: 2014-09-01 12:00:00
   portable_data_hash: 0b9a7787660e1fce4a93f33e01376ba6+81
   manifest_text: ". cdd549ae79fe6640fa3d5c6261d8303c+195 0:195:zzzzz-8i9sb-0vsrcqi7whchuil.log.txt\n"
   name: real_log_collection
@@ -278,6 +279,7 @@ collection_owned_by_foo:
   manifest_text: ". 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz\n"
   owner_uuid: zzzzz-tpzed-81hsbo6mk8nl05c
   created_at: 2014-02-03T17:22:54Z
+  modified_at: 2014-02-03T17:22:54Z
   name: collection_owned_by_foo
 
 collection_to_remove_from_subproject:
@@ -287,6 +289,7 @@ collection_to_remove_from_subproject:
   manifest_text: ". 8258b505536a9ab47baa2f4281cb932a+9 0:9:missingno\n"
   owner_uuid: zzzzz-j7d0g-axqo7eu9pwvna1x
   created_at: 2014-10-15T10:45:00
+  modified_at: 2014-10-15T10:45:00
   name: Collection to remove from subproject
 
 collection_with_files_in_subdir:
@@ -307,6 +310,8 @@ graph_test_collection1:
   portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
   manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
   name: bar_file
+  created_at: 2014-02-03T17:22:54Z
+  modified_at: 2014-02-03T17:22:54Z
 
 graph_test_collection2:
   uuid: zzzzz-4zz18-uukreo9rbgwsujx
@@ -314,6 +319,8 @@ graph_test_collection2:
   portable_data_hash: 65b17c95fdbc9800fc48acda4e9dcd0b+93
   manifest_text: ". 6a4ff0499484c6c79c95cd8c566bd25f+249025 0:249025:FOO_General_Public_License,_version_3.pdf\n"
   name: "FOO General Public License, version 3"
+  created_at: 2014-02-03T17:22:54Z
+  modified_at: 2014-02-03T17:22:54Z
 
 graph_test_collection3:
   uuid: zzzzz-4zz18-uukreo9rbgwsujj
@@ -321,6 +328,8 @@ graph_test_collection3:
   portable_data_hash: ea10d51bcf88862dbcc36eb292017dfd+45
   manifest_text: ". 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz\n"
   name: "baz file"
+  created_at: 2014-02-03T17:22:54Z
+  modified_at: 2014-02-03T17:22:54Z
 
 collection_1_owned_by_fuse:
   uuid: zzzzz-4zz18-ovx05bfzormx3bg
@@ -557,6 +566,7 @@ collection_<%=i%>_of_10:
   uuid: zzzzz-4zz18-10gneyn6brkx<%= i.to_s.rjust(3, '0') %>
   owner_uuid: zzzzz-j7d0g-0010collections
   created_at: <%= i.minute.ago.to_s(:db) %>
+  modified_at: <%= i.minute.ago.to_s(:db) %>
 <% end %>
 
 # collections in project_with_201_collections
@@ -568,6 +578,7 @@ collection_<%=i%>_of_201:
   uuid: zzzzz-4zz18-201gneyn6brd<%= i.to_s.rjust(3, '0') %>
   owner_uuid: zzzzz-j7d0g-0201collections
   created_at: <%= i.minute.ago.to_s(:db) %>
+  modified_at: <%= i.minute.ago.to_s(:db) %>
 <% end %>
 
 # Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper

diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index f0a0f3f1aab62f1fa6367cff900eac261f22157b..9b7eb7543a4ebed086aba2d409f44fcc789ef222 100644 (file)
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -247,6 +247,30 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections, err e
                }
        }
 
+       // Make one final API request to verify that we have processed all collections available up to the latest modification date
+       var collections SdkCollectionList
+       sdkParams["filters"].([][]string)[0][1] = "<="
+       sdkParams["limit"] = 0
+       err = params.Client.List("collections", sdkParams, &collections)
+       if err != nil {
+               return
+       }
+       finalNumberOfCollectionsAvailable, err :=
+               util.NumberItemsAvailable(params.Client, "collections")
+       if err != nil {
+               return
+       }
+       if totalCollections < finalNumberOfCollectionsAvailable {
+               err = fmt.Errorf("API server indicates a total of %d collections "+
+                               "available up to %v, but we only retrieved %d. "+
+                               "Refusing to continue as this could indicate an "+
+                               "otherwise undetected failure.",
+                               finalNumberOfCollectionsAvailable, 
+                               sdkParams["filters"].([][]string)[0][2],
+                               totalCollections)
+               return
+       }
+
        // Write the heap profile for examining memory usage
        err = WriteHeapProfile()