use multiple store engines, backward compatible

Change-Id: I8e650c15a359eab176b1a1d7d16e8014bd0826e1
diff --git a/README.rst b/README.rst
index 7fa9cd9..50af4ff 100644
--- a/README.rst
+++ b/README.rst
@@ -134,7 +134,7 @@
         enabled: true
         version: juno
         storage:
-          engine: rbd
+          engine: rbd,http
           user: glance
           pool: images
           chunk_size: 8
diff --git a/glance/files/liberty/glance-api-paste.ini b/glance/files/liberty/glance-api-paste.ini
index 4480f62..c809673 100644
--- a/glance/files/liberty/glance-api-paste.ini
+++ b/glance/files/liberty/glance-api-paste.ini
@@ -1,39 +1,39 @@
 {%- from "glance/map.jinja" import server with context %}
 # Use this pipeline for no auth or image caching - DEFAULT
 [pipeline:glance-api]
-pipeline = healthcheck versionnegotiation osprofiler unauthenticated-context {% if server.audit.enabled %}audit {% endif%}rootapp
+pipeline = healthcheck versionnegotiation osprofiler unauthenticated-context {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for image caching and no auth
 [pipeline:glance-api-caching]
-pipeline = healthcheck versionnegotiation osprofiler unauthenticated-context cache {% if server.audit.enabled %}audit {% endif%}rootapp
+pipeline = healthcheck versionnegotiation osprofiler unauthenticated-context cache {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for caching w/ management interface but no auth
 [pipeline:glance-api-cachemanagement]
-pipeline = healthcheck versionnegotiation osprofiler unauthenticated-context cache cachemanage {% if server.audit.enabled %}audit {% endif%}rootapp
+pipeline = healthcheck versionnegotiation osprofiler unauthenticated-context cache cachemanage {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for keystone auth
 [pipeline:glance-api-keystone]
-pipeline = healthcheck versionnegotiation osprofiler authtoken context {% if server.audit.enabled %}audit {% endif%}rootapp
+pipeline = healthcheck versionnegotiation osprofiler authtoken context {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for keystone auth with image caching
 [pipeline:glance-api-keystone+caching]
-pipeline = healthcheck versionnegotiation osprofiler authtoken context cache {% if server.audit.enabled %}audit {% endif%}rootapp
+pipeline = healthcheck versionnegotiation osprofiler authtoken context cache {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for keystone auth with caching and cache management
 [pipeline:glance-api-keystone+cachemanagement]
-pipeline = healthcheck versionnegotiation osprofiler authtoken context cache cachemanage {% if server.audit.enabled %}audit {% endif%}rootapp
+pipeline = healthcheck versionnegotiation osprofiler authtoken context cache cachemanage {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for authZ only. This means that the registry will treat a
 # user as authenticated without making requests to keystone to reauthenticate
 # the user.
 [pipeline:glance-api-trusted-auth]
-pipeline = healthcheck versionnegotiation osprofiler context {% if server.audit.enabled %}audit {% endif%}rootapp
+pipeline = healthcheck versionnegotiation osprofiler context {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for authZ only. This means that the registry will treat a
 # user as authenticated without making requests to keystone to reauthenticate
 # the user and uses cache management
 [pipeline:glance-api-trusted-auth+cachemanagement]
-pipeline = healthcheck versionnegotiation osprofiler context cache cachemanage {% if server.audit.enabled %}audit {% endif%}rootapp
+pipeline = healthcheck versionnegotiation osprofiler context cache cachemanage {% if server.audit.enabled %}audit {% endif %}rootapp
 
 [composite:rootapp]
 paste.composite_factory = glance.api:root_app_factory
diff --git a/glance/files/liberty/glance-api.conf.Debian b/glance/files/liberty/glance-api.conf.Debian
index daf8f1c..6dad168 100644
--- a/glance/files/liberty/glance-api.conf.Debian
+++ b/glance/files/liberty/glance-api.conf.Debian
@@ -1,4 +1,5 @@
 {%- from "glance/map.jinja" import server with context %}
+{% set storage_engines = server.storage.engine.split(',') %}
 [DEFAULT]
 # Show more verbose log output (sets INFO log level output)
 #verbose = False
@@ -577,12 +578,12 @@
 # by the stores option.
 # Deprecated group/name - [DEFAULT]/default_store
 # Default: 'file'
-{%- if server.storage.engine == 'file' %}
+{%- if 'file' in storage_engines %}
 default_store = file
 stores = glance.store.filesystem.Store,
          glance.store.http.Store
 {%- else %}
-default_store = {{ server.storage.engine }}
+default_store = {{ storage_engines[0]  }}
 stores = {{ server.storage.engine }}
 {%- endif %}
 # ============ Filesystem Store Options ========================
@@ -765,7 +766,7 @@
 
 # ============ RBD Store Options =============================
 
-{%- if server.storage.engine == 'rbd' %}
+{%- if 'rbd' in storage_engines %}
 # ============ RBD Store Options =============================
 
 # Ceph configuration file path
diff --git a/glance/files/mitaka/glance-api-paste.ini b/glance/files/mitaka/glance-api-paste.ini
index 5564493..d2e33c0 100644
--- a/glance/files/mitaka/glance-api-paste.ini
+++ b/glance/files/mitaka/glance-api-paste.ini
@@ -1,39 +1,39 @@
 {%- from "glance/map.jinja" import server with context %}
 # Use this pipeline for no auth or image caching - DEFAULT
 [pipeline:glance-api]
-pipeline = cors healthcheck versionnegotiation osprofiler unauthenticated-context {% if server.audit.enabled %}audit {% endif% }rootapp
+pipeline = cors healthcheck versionnegotiation osprofiler unauthenticated-context {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for image caching and no auth
 [pipeline:glance-api-caching]
-pipeline = cors healthcheck versionnegotiation osprofiler unauthenticated-context cache {% if server.audit.enabled %}audit {% endif% }rootapp
+pipeline = cors healthcheck versionnegotiation osprofiler unauthenticated-context cache {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for caching w/ management interface but no auth
 [pipeline:glance-api-cachemanagement]
-pipeline = cors healthcheck versionnegotiation osprofiler unauthenticated-context cache cachemanage {% if server.audit.enabled %}audit {% endif% }rootapp
+pipeline = cors healthcheck versionnegotiation osprofiler unauthenticated-context cache cachemanage {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for keystone auth
 [pipeline:glance-api-keystone]
-pipeline = cors healthcheck versionnegotiation osprofiler authtoken context  {% if server.audit.enabled %}audit {% endif% }rootapp
+pipeline = cors healthcheck versionnegotiation osprofiler authtoken context  {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for keystone auth with image caching
 [pipeline:glance-api-keystone+caching]
-pipeline = cors healthcheck versionnegotiation osprofiler authtoken context cache{% if server.audit.enabled %}audit {% endif% }rootapp
+pipeline = cors healthcheck versionnegotiation osprofiler authtoken context cache {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for keystone auth with caching and cache management
 [pipeline:glance-api-keystone+cachemanagement]
-pipeline = cors healthcheck versionnegotiation osprofiler authtoken context cache cachemanage {% if server.audit.enabled %}audit {% endif% }rootapp
+pipeline = cors healthcheck versionnegotiation osprofiler authtoken context cache cachemanage {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for authZ only. This means that the registry will treat a
 # user as authenticated without making requests to keystone to reauthenticate
 # the user.
 [pipeline:glance-api-trusted-auth]
-pipeline = cors healthcheck versionnegotiation osprofiler context {% if server.audit.enabled %}audit {% endif% }rootapp
+pipeline = cors healthcheck versionnegotiation osprofiler context {% if server.audit.enabled %}audit {% endif %}rootapp
 
 # Use this pipeline for authZ only. This means that the registry will treat a
 # user as authenticated without making requests to keystone to reauthenticate
 # the user and uses cache management
 [pipeline:glance-api-trusted-auth+cachemanagement]
-pipeline = cors healthcheck versionnegotiation osprofiler context cache cachemanage {% if server.audit.enabled %}audit {% endif% }rootapp
+pipeline = cors healthcheck versionnegotiation osprofiler context cache cachemanage {% if server.audit.enabled %}audit {% endif %}rootapp
 
 [composite:rootapp]
 paste.composite_factory = glance.api:root_app_factory
diff --git a/glance/files/mitaka/glance-api.conf.Debian b/glance/files/mitaka/glance-api.conf.Debian
index e961239..d0055a4 100644
--- a/glance/files/mitaka/glance-api.conf.Debian
+++ b/glance/files/mitaka/glance-api.conf.Debian
@@ -1,4 +1,5 @@
 {%- from "glance/map.jinja" import server with context %}
+{% set storage_engines = server.storage.engine.split(',') %}
 [DEFAULT]
 
 #
@@ -761,11 +762,11 @@
 # List of stores enabled. Valid stores are: cinder, file, http, rbd,
 # sheepdog, swift, s3, vsphere (list value)
 #stores = file,http
-{%- if server.storage.engine == 'file' %}
+{%- if 'file' in storage_engines %}
 default_store = file
 stores = file,http
 {%- else %}
-default_store = {{ server.storage.engine }}
+default_store = {{ storage_engines[0]  }}
 stores = {{ server.storage.engine }}
 {%- endif %}
 # Default scheme to use to store image data. The scheme must be
@@ -913,7 +914,7 @@
 # The config file that has the swift account(s)configs. (string value)
 #swift_store_config_file = <None>
 
-{%- if server.storage.engine == 'rbd' %}
+{%- if 'rbd' in storage_engines %}
 # RADOS images will be chunked into objects of this size (in
 # megabytes). For best performance, this should be a power of two.
 # (integer value)