AMAZON_AWS_S3_PKG - get_object_tab to return entire list (enhancement)

[GoogleCodeExporter]

I've rewritten get_object_list as an internal procedure that uses the "marker" 
parameter, so that get_object_tab can now call the Amazon API multiple times to 
return the complete set of objects.

The get_object_list function remains functionally unchanged in this version - 
it just returns one set of objects - it could be enhanced to support the marker 
parameter as well, I guess, but I'd rather not expose that sort of thing to the 
caller personally.

The nice thing about the pipelined function is that the subsequent calls to 
Amazon will only be executed if the client actually fetches all the rows.

------

-- private procedure, used by the functions get_object_list and get_object_tab
procedure get_object_list (p_bucket_name in varchar2,
                           p_prefix in varchar2,
                           p_max_keys in number,
                           p_list out t_object_list,
                           p_more_marker in out varchar2)
as
  l_clob                         clob;
  l_xml                          xmltype;

  l_date_str                     varchar2(255);
  l_auth_str                     varchar2(255);

  l_header_names                 t_str_array := t_str_array();
  l_header_values                t_str_array := t_str_array();

  l_count                        pls_integer := 0;
  l_returnvalue                  t_object_list;

begin

  /*

  Purpose:   get objects

  Remarks:   see http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?RESTObjectGET.html

  Who     Date        Description
  ------  ----------  -------------------------------------
  MBR     15.01.2011  Created

  */

  l_date_str := amazon_aws_auth_pkg.get_date_string;
  l_auth_str := amazon_aws_auth_pkg.get_auth_string ('GET' || chr(10) || chr(10) || chr(10) || l_date_str || chr(10) || '/' || p_bucket_name || '/');

  l_header_names.extend;
  l_header_names(1) := 'Host';
  l_header_values.extend;
  l_header_values(1) := get_host (p_bucket_name);

  l_header_names.extend;
  l_header_names(2) := 'Date';
  l_header_values.extend;
  l_header_values(2) := l_date_str;

  l_header_names.extend;
  l_header_names(3) := 'Authorization';
  l_header_values.extend;
  l_header_values(3) := l_auth_str;

  if p_more_marker is not null then
    l_clob := make_request (get_url(p_bucket_name) || '?marker=' || p_more_marker || '&max-keys=' || p_max_keys || '&prefix=' || utl_url.escape(p_prefix), 'GET', l_header_names, l_header_values, null);
  else
    l_clob := make_request (get_url(p_bucket_name) || '?max-keys=' || p_max_keys || '&prefix=' || utl_url.escape(p_prefix), 'GET', l_header_names, l_header_values, null);
  end if;

  if (l_clob is not null) and (length(l_clob) > 0) then

    l_xml := xmltype (l_clob);

    check_for_errors (l_xml);

    for l_rec in (
      select extractValue(value(t), '*/Key', g_aws_namespace_s3_full) as key,
        extractValue(value(t), '*/Size', g_aws_namespace_s3_full) as size_bytes,
        extractValue(value(t), '*/LastModified', g_aws_namespace_s3_full) as last_modified
      from table(xmlsequence(l_xml.extract('//ListBucketResult/Contents', g_aws_namespace_s3_full))) t
      ) loop
      l_count := l_count + 1;
      l_returnvalue(l_count).key := l_rec.key;
      l_returnvalue(l_count).size_bytes := l_rec.size_bytes;
      l_returnvalue(l_count).last_modified := to_date(l_rec.last_modified, g_date_format_xml);
    end loop;

    -- check if this is the last set of data or not

    l_xml := l_xml.extract('//ListBucketResult/IsTruncated/text()', g_aws_namespace_s3_full);

    if l_xml is not null and l_xml.getStringVal = 'true' then
      p_more_marker := l_returnvalue(l_returnvalue.LAST).key;
    end if;

  end if;

  p_list := l_returnvalue;

end get_object_list;

function get_object_list (p_bucket_name in varchar2,
                          p_prefix in varchar2 := null,
                          p_max_keys in number := null) return t_object_list
as
  l_object_list                  t_object_list;
  l_more_marker                  varchar2(4000);
begin
  /*

  Purpose:   get objects

  Remarks:   see http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?RESTObjectGET.html

  Who     Date        Description
  ------  ----------  -------------------------------------
  JKEMP   14.08.2012  Created

  */

  get_object_list
    (p_bucket_name => p_bucket_name
    ,p_prefix      => p_prefix
    ,p_max_keys    => p_max_keys
    ,p_list        => l_object_list
    ,p_more_marker => l_more_marker --ignored by this function
    );

  return l_object_list;

end get_object_list;

function get_object_tab (p_bucket_name in varchar2,
                         p_prefix in varchar2 := null,
                         p_max_keys in number := null) return t_object_tab pipelined
as
  l_object_list                  t_object_list;
  l_more_marker                  varchar2(4000);
begin

  /*

  Purpose:   get objects

  Remarks:

  Who     Date        Description
  ------  ----------  -------------------------------------
  MBR     19.01.2011  Created

  */

  loop

    get_object_list
      (p_bucket_name => p_bucket_name
      ,p_prefix      => p_prefix
      ,p_max_keys    => p_max_keys
      ,p_list        => l_object_list
      ,p_more_marker => l_more_marker
      );

    for i in 1 .. l_object_list.count loop
      pipe row (l_object_list(i));
    end loop;

    exit when l_more_marker is null;

  end loop;

  return;

end get_object_tab;

Original issue reported on code.google.com by jeffrey....@jk64.com on 16 Aug 2012 at 12:38

[GoogleCodeExporter]

Original comment by thehunge...@gmail.com on 16 Aug 2012 at 2:20

• Changed state: Accepted
• Added labels: Type-Enhancement
• Removed labels: Type-Defect

[GoogleCodeExporter]

Feature implemented in latest version of library.

Original comment by thehunge...@gmail.com on 17 Feb 2013 at 9:21

• Changed state: Fixed