Merge pull request #377 from jrperritt/get-all-pages
Get All Pages; Closes #298
diff --git a/acceptance/openstack/objectstorage/v1/containers_test.go b/acceptance/openstack/objectstorage/v1/containers_test.go
index d6832f1..8328a4f 100644
--- a/acceptance/openstack/objectstorage/v1/containers_test.go
+++ b/acceptance/openstack/objectstorage/v1/containers_test.go
@@ -87,3 +87,51 @@
}
}
}
+
+func TestListAllContainers(t *testing.T) {
+ // Create a new client to execute the HTTP requests. See common.go for newClient body.
+ client := newClient(t)
+
+ numContainers := 20
+
+ // Create a slice of random container names.
+ cNames := make([]string, numContainers)
+ for i := 0; i < numContainers; i++ {
+ cNames[i] = tools.RandomString("gophercloud-test-container-", 8)
+ }
+
+ // Create numContainers containers.
+ for i := 0; i < len(cNames); i++ {
+ res := containers.Create(client, cNames[i], nil)
+ th.AssertNoErr(t, res.Err)
+ }
+ // Delete the numContainers containers after function completion.
+ defer func() {
+ for i := 0; i < len(cNames); i++ {
+ res := containers.Delete(client, cNames[i])
+ th.AssertNoErr(t, res.Err)
+ }
+ }()
+
+ // List all the numContainer names that were just created. To just list those,
+ // the 'prefix' parameter is used.
+ allPages, err := containers.List(client, &containers.ListOpts{Full: true, Limit: 5, Prefix: "gophercloud-test-container-"}).AllPages()
+ th.AssertNoErr(t, err)
+ containerInfoList, err := containers.ExtractInfo(allPages)
+ th.AssertNoErr(t, err)
+ for _, n := range containerInfoList {
+ t.Logf("Container: Name [%s] Count [%d] Bytes [%d]",
+ n.Name, n.Count, n.Bytes)
+ }
+ th.AssertEquals(t, numContainers, len(containerInfoList))
+
+ // List the info for all the numContainer containers that were created.
+ allPages, err = containers.List(client, &containers.ListOpts{Full: false, Limit: 2, Prefix: "gophercloud-test-container-"}).AllPages()
+ th.AssertNoErr(t, err)
+ containerNamesList, err := containers.ExtractNames(allPages)
+ th.AssertNoErr(t, err)
+ for _, n := range containerNamesList {
+ t.Logf("Container: Name [%s]", n)
+ }
+ th.AssertEquals(t, numContainers, len(containerNamesList))
+}
diff --git a/openstack/blockstorage/v1/volumes/requests.go b/openstack/blockstorage/v1/volumes/requests.go
index e620ca6..e67ba10 100644
--- a/openstack/blockstorage/v1/volumes/requests.go
+++ b/openstack/blockstorage/v1/volumes/requests.go
@@ -152,6 +152,7 @@
createPage := func(r pagination.PageResult) pagination.Page {
return ListResult{pagination.SinglePageBase(r)}
}
+
return pagination.NewPager(client, url, createPage)
}
diff --git a/openstack/blockstorage/v1/volumes/requests_test.go b/openstack/blockstorage/v1/volumes/requests_test.go
index 7643375..c484cf0 100644
--- a/openstack/blockstorage/v1/volumes/requests_test.go
+++ b/openstack/blockstorage/v1/volumes/requests_test.go
@@ -45,6 +45,32 @@
}
}
+func TestListAll(t *testing.T) {
+ th.SetupHTTP()
+ defer th.TeardownHTTP()
+
+ MockListResponse(t)
+
+ allPages, err := List(client.ServiceClient(), &ListOpts{}).AllPages()
+ th.AssertNoErr(t, err)
+ actual, err := ExtractVolumes(allPages)
+ th.AssertNoErr(t, err)
+
+ expected := []Volume{
+ Volume{
+ ID: "289da7f8-6440-407c-9fb4-7db01ec49164",
+ Name: "vol-001",
+ },
+ Volume{
+ ID: "96c3bda7-c82a-4f50-be73-ca7621794835",
+ Name: "vol-002",
+ },
+ }
+
+ th.CheckDeepEquals(t, expected, actual)
+
+}
+
func TestGet(t *testing.T) {
th.SetupHTTP()
defer th.TeardownHTTP()
diff --git a/openstack/compute/v2/servers/requests_test.go b/openstack/compute/v2/servers/requests_test.go
index 017e793..253a179 100644
--- a/openstack/compute/v2/servers/requests_test.go
+++ b/openstack/compute/v2/servers/requests_test.go
@@ -39,6 +39,19 @@
}
}
+func TestListAllServers(t *testing.T) {
+ th.SetupHTTP()
+ defer th.TeardownHTTP()
+ HandleServerListSuccessfully(t)
+
+ allPages, err := List(client.ServiceClient(), ListOpts{}).AllPages()
+ th.AssertNoErr(t, err)
+ actual, err := ExtractServers(allPages)
+ th.AssertNoErr(t, err)
+ th.CheckDeepEquals(t, ServerHerp, actual[0])
+ th.CheckDeepEquals(t, ServerDerp, actual[1])
+}
+
func TestCreateServer(t *testing.T) {
th.SetupHTTP()
defer th.TeardownHTTP()
diff --git a/openstack/objectstorage/v1/containers/fixtures.go b/openstack/objectstorage/v1/containers/fixtures.go
index 9c84bce..e607352 100644
--- a/openstack/objectstorage/v1/containers/fixtures.go
+++ b/openstack/objectstorage/v1/containers/fixtures.go
@@ -55,6 +55,14 @@
"name": "marktwain"
}
]`)
+ case "janeausten":
+ fmt.Fprintf(w, `[
+ {
+ "count": 1,
+ "bytes": 14,
+ "name": "marktwain"
+ }
+ ]`)
case "marktwain":
fmt.Fprintf(w, `[]`)
default:
@@ -77,6 +85,8 @@
switch marker {
case "":
fmt.Fprintf(w, "janeausten\nmarktwain\n")
+ case "janeausten":
+ fmt.Fprintf(w, "marktwain\n")
case "marktwain":
fmt.Fprintf(w, ``)
default:
diff --git a/openstack/objectstorage/v1/containers/requests_test.go b/openstack/objectstorage/v1/containers/requests_test.go
index f650696..0ccd5a7 100644
--- a/openstack/objectstorage/v1/containers/requests_test.go
+++ b/openstack/objectstorage/v1/containers/requests_test.go
@@ -29,6 +29,18 @@
th.CheckEquals(t, count, 1)
}
+func TestListAllContainerInfo(t *testing.T) {
+ th.SetupHTTP()
+ defer th.TeardownHTTP()
+ HandleListContainerInfoSuccessfully(t)
+
+ allPages, err := List(fake.ServiceClient(), &ListOpts{Full: true}).AllPages()
+ th.AssertNoErr(t, err)
+ actual, err := ExtractInfo(allPages)
+ th.AssertNoErr(t, err)
+ th.CheckDeepEquals(t, ExpectedListInfo, actual)
+}
+
func TestListContainerNames(t *testing.T) {
th.SetupHTTP()
defer th.TeardownHTTP()
@@ -51,6 +63,18 @@
th.CheckEquals(t, count, 1)
}
+func TestListAllContainerNames(t *testing.T) {
+ th.SetupHTTP()
+ defer th.TeardownHTTP()
+ HandleListContainerNamesSuccessfully(t)
+
+ allPages, err := List(fake.ServiceClient(), &ListOpts{Full: false}).AllPages()
+ th.AssertNoErr(t, err)
+ actual, err := ExtractNames(allPages)
+ th.AssertNoErr(t, err)
+ th.CheckDeepEquals(t, ExpectedListNames, actual)
+}
+
func TestCreateContainer(t *testing.T) {
th.SetupHTTP()
defer th.TeardownHTTP()
diff --git a/pagination/linked.go b/pagination/linked.go
index 461fa49..e9bd8de 100644
--- a/pagination/linked.go
+++ b/pagination/linked.go
@@ -59,3 +59,9 @@
}
}
}
+
+// GetBody returns the linked page's body. This method is needed to satisfy the
+// Page interface.
+func (current LinkedPageBase) GetBody() interface{} {
+ return current.Body
+}
diff --git a/pagination/linked_test.go b/pagination/linked_test.go
index 4d3248e..1ac0f73 100644
--- a/pagination/linked_test.go
+++ b/pagination/linked_test.go
@@ -105,3 +105,16 @@
t.Errorf("Expected 3 calls, but was %d", callCount)
}
}
+
+func TestAllPagesLinked(t *testing.T) {
+ pager := createLinked(t)
+ defer testhelper.TeardownHTTP()
+
+ page, err := pager.AllPages()
+ testhelper.AssertNoErr(t, err)
+
+ expected := []int{1, 2, 3, 4, 5, 6, 7, 8, 9}
+ actual, err := ExtractLinkedInts(page)
+ testhelper.AssertNoErr(t, err)
+ testhelper.CheckDeepEquals(t, expected, actual)
+}
diff --git a/pagination/marker.go b/pagination/marker.go
index e7688c2..f355afc 100644
--- a/pagination/marker.go
+++ b/pagination/marker.go
@@ -32,3 +32,9 @@
return currentURL.String(), nil
}
+
+// GetBody returns the linked page's body. This method is needed to satisfy the
+// Page interface.
+func (current MarkerPageBase) GetBody() interface{} {
+ return current.Body
+}
diff --git a/pagination/marker_test.go b/pagination/marker_test.go
index 3b1df1d..f4d55be 100644
--- a/pagination/marker_test.go
+++ b/pagination/marker_test.go
@@ -111,3 +111,16 @@
testhelper.AssertNoErr(t, err)
testhelper.AssertEquals(t, callCount, 3)
}
+
+func TestAllPagesMarker(t *testing.T) {
+ pager := createMarkerPaged(t)
+ defer testhelper.TeardownHTTP()
+
+ page, err := pager.AllPages()
+ testhelper.AssertNoErr(t, err)
+
+ expected := []string{"aaa", "bbb", "ccc", "ddd", "eee", "fff", "ggg", "hhh", "iii"}
+ actual, err := ExtractMarkerStrings(page)
+ testhelper.AssertNoErr(t, err)
+ testhelper.CheckDeepEquals(t, expected, actual)
+}
diff --git a/pagination/pager.go b/pagination/pager.go
index 5c20e16..ea47c69 100644
--- a/pagination/pager.go
+++ b/pagination/pager.go
@@ -2,6 +2,10 @@
import (
"errors"
+ "fmt"
+ "net/http"
+ "reflect"
+ "strings"
"github.com/rackspace/gophercloud"
)
@@ -25,6 +29,9 @@
// IsEmpty returns true if this Page has no items in it.
IsEmpty() (bool, error)
+
+ // GetBody returns the Page Body. This is used in the `AllPages` method.
+ GetBody() interface{}
}
// Pager knows how to advance through a specific resource collection, one page at a time.
@@ -113,3 +120,105 @@
}
}
}
+
+// AllPages returns all the pages from a `List` operation in a single page,
+// allowing the user to retrieve all the pages at once.
+func (p Pager) AllPages() (Page, error) {
+ // pagesSlice holds all the pages until they get converted into as Page Body.
+ var pagesSlice []interface{}
+ // body will contain the final concatenated Page body.
+ var body reflect.Value
+
+ // Grab a test page to ascertain the page body type.
+ testPage, err := p.fetchNextPage(p.initialURL)
+ if err != nil {
+ return nil, err
+ }
+ // Store the page type so we can use reflection to create a new mega-page of
+ // that type.
+ pageType := reflect.TypeOf(testPage)
+
+ // Switch on the page body type. Recognized types are `map[string]interface{}`,
+ // `[]byte`, and `[]interface{}`.
+ switch testPage.GetBody().(type) {
+ case map[string]interface{}:
+ // key is the map key for the page body if the body type is `map[string]interface{}`.
+ var key string
+ // Iterate over the pages to concatenate the bodies.
+ err := p.EachPage(func(page Page) (bool, error) {
+ b := page.GetBody().(map[string]interface{})
+ for k := range b {
+ // If it's a linked page, we don't want the `links`, we want the other one.
+ if !strings.HasSuffix(k, "links") {
+ key = k
+ }
+ }
+ pagesSlice = append(pagesSlice, b[key].([]interface{})...)
+ return true, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ // Set body to value of type `map[string]interface{}`
+ body = reflect.MakeMap(reflect.MapOf(reflect.TypeOf(key), reflect.TypeOf(pagesSlice)))
+ body.SetMapIndex(reflect.ValueOf(key), reflect.ValueOf(pagesSlice))
+ case []byte:
+ // Iterate over the pages to concatenate the bodies.
+ err := p.EachPage(func(page Page) (bool, error) {
+ b := page.GetBody().([]byte)
+ pagesSlice = append(pagesSlice, b)
+ // seperate pages with a comma
+ pagesSlice = append(pagesSlice, []byte{10})
+ return true, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ // Remove the trailing comma.
+ pagesSlice = pagesSlice[:len(pagesSlice)-1]
+ var b []byte
+ // Combine the slice of slices in to a single slice.
+ for _, slice := range pagesSlice {
+ b = append(b, slice.([]byte)...)
+ }
+ // Set body to value of type `bytes`.
+ body = reflect.New(reflect.TypeOf(b)).Elem()
+ body.SetBytes(b)
+ case []interface{}:
+ // Iterate over the pages to concatenate the bodies.
+ err := p.EachPage(func(page Page) (bool, error) {
+ b := page.GetBody().([]interface{})
+ pagesSlice = append(pagesSlice, b...)
+ return true, nil
+ })
+ if err != nil {
+ return nil, err
+ }
+ // Set body to value of type `[]interface{}`
+ body = reflect.MakeSlice(reflect.TypeOf(pagesSlice), len(pagesSlice), len(pagesSlice))
+ for i, s := range pagesSlice {
+ body.Index(i).Set(reflect.ValueOf(s))
+ }
+ default:
+ return nil, fmt.Errorf("Page body has unrecognized type.")
+ }
+
+ // Each `Extract*` function is expecting a specific type of page coming back,
+ // otherwise the type assertion in those functions will fail. pageType is needed
+ // to create a type in this method that has the same type that the `Extract*`
+ // function is expecting and set the Body of that object to the concatenated
+ // pages.
+ page := reflect.New(pageType)
+ // Set the page body to be the concatenated pages.
+ page.Elem().FieldByName("Body").Set(body)
+ // Set any additional headers that were pass along. The `objectstorage` pacakge,
+ // for example, passes a Content-Type header.
+ h := make(http.Header)
+ for k, v := range p.Headers {
+ h.Add(k, v)
+ }
+ page.Elem().FieldByName("Header").Set(reflect.ValueOf(h))
+ // Type assert the page to a Page interface so that the type assertion in the
+ // `Extract*` methods will work.
+ return page.Elem().Interface().(Page), err
+}
diff --git a/pagination/single.go b/pagination/single.go
index 4dd3c5c..f78d4ab 100644
--- a/pagination/single.go
+++ b/pagination/single.go
@@ -7,3 +7,9 @@
func (current SinglePageBase) NextPageURL() (string, error) {
return "", nil
}
+
+// GetBody returns the single page's body. This method is needed to satisfy the
+// Page interface.
+func (current SinglePageBase) GetBody() interface{} {
+ return current.Body
+}
diff --git a/pagination/single_test.go b/pagination/single_test.go
index 8817d57..4af0fee 100644
--- a/pagination/single_test.go
+++ b/pagination/single_test.go
@@ -69,3 +69,16 @@
testhelper.CheckNoErr(t, err)
testhelper.CheckEquals(t, 1, callCount)
}
+
+func TestAllPagesSingle(t *testing.T) {
+ pager := setupSinglePaged()
+ defer testhelper.TeardownHTTP()
+
+ page, err := pager.AllPages()
+ testhelper.AssertNoErr(t, err)
+
+ expected := []int{1, 2, 3}
+ actual, err := ExtractSingleInts(page)
+ testhelper.AssertNoErr(t, err)
+ testhelper.CheckDeepEquals(t, expected, actual)
+}