Group by groups

I have a lot of data that I need to transform and "group of groups". Let me explain with an example:

SQL> create table orig_data as
  2  select distinct job, deptno
  3  from scott.emp e
  4  /

Table created.

SQL> select job
  2       , deptno
  3    from orig_data
  4   order by
  5         job
  6       , deptno
  7  /

JOB           DEPTNO
--------- ----------
ANALYST           20
CLERK             10
CLERK             20
CLERK             30
MANAGER           10
MANAGER           20
MANAGER           30
PRESIDENT         10
SALESMAN          30

9 rows selected.

Real-world data about 5 million lines.

I have a working group (I use xmlagg here because I'm on version 11.1 and therefore no listagg ;-))):

SQL> select od.job
  2       , rtrim(xmlagg(xmlelement(d,od.deptno,',').extract('//text()') order by od.deptno),',') deptnos
  3    from orig_data od
  4   group by od.job
  5  /

JOB       DEPTNOS
--------- ------------------------------
ANALYST   20
CLERK     10,20,30
MANAGER   10,20,30
PRESIDENT 10
SALESMAN  30

I note here that the two jobs CLERK and MANAGER has the same set of deptnos.
So if I group by deptnos I get this result:

SQL> select s2.deptnos
  2       , rtrim(xmlagg(xmlelement(j,s2.job,',').extract('//text()') order by s2.job),',') jobs
  3    from (
  4     select od.job
  5          , rtrim(xmlagg(xmlelement(d,od.deptno,',').extract('//text()') order by od.deptno),',') deptnos
  6       from orig_data od
  7      group by od.job
  8         ) s2
  9   group by s2.deptnos
 10  /

DEPTNOS                        JOBS
------------------------------ ------------------------------
10                             PRESIDENT
10,20,30                       CLERK,MANAGER
20                             ANALYST
30                             SALESMAN

My requirement is to identify all of these unique groups of deptnos in my table of orig_data, give each group as a surrogate key in a parent table and then populate two tables of children with the deptnos of each group and the jobs that have this group of deptnos:

SQL> create table groups (
  2     groupkey number primary key
  3  )
  4  /

Table created.

SQL> create table groups_depts (
  2     groupkey number references groups (groupkey)
  3   , deptno number(2)
  4  )
  5  /

Table created.

SQL> create table groups_jobs (
  2     groupkey number references groups (groupkey)
  3   , job varchar2(9)
  4  )
  5  /

Table created.

For the substitution groupkey I can use a rownumber on my group by query deptnos:

SQL> select row_number() over (order by s2.deptnos) groupkey
  2       , s2.deptnos
  3       , rtrim(xmlagg(xmlelement(j,s2.job,',').extract('//text()') order by s2.job),',') jobs
  4    from (
  5     select od.job
  6          , rtrim(xmlagg(xmlelement(d,od.deptno,',').extract('//text()') order by od.deptno),',') deptnos
  7       from orig_data od
  8      group by od.job
  9         ) s2
 10   group by s2.deptnos
 11  /

  GROUPKEY DEPTNOS                        JOBS
---------- ------------------------------ ------------------------------
         1 10                             PRESIDENT
         2 10,20,30                       CLERK,MANAGER
         3 20                             ANALYST
         4 30                             SALESMAN

This application, that I can use for a (slow) insert in my three tables in this simple way:

SQL> begin
  2     for g in (
  3        select row_number() over (order by s2.deptnos) groupkey
  4             , s2.deptnos
  5             , rtrim(xmlagg(xmlelement(j,s2.job,',').extract('//text()') order by s2.job),',') jobs
  6          from (
  7           select od.job
  8                , rtrim(xmlagg(xmlelement(d,od.deptno,',').extract('//text()') order by od.deptno),',') deptnos
  9             from orig_data od
 10            group by od.job
 11               ) s2
 12         group by s2.deptnos
 13     ) loop
 14        insert into groups values (g.groupkey);
 15
 16        insert into groups_depts
 17           select g.groupkey
 18                , to_number(regexp_substr(str, '[^,]+', 1, level)) deptno
 19             from (
 20                    select rownum id
 21                         , g.deptnos str
 22                      from dual
 23                  )
 24           connect by instr(str, ',', 1, level-1) > 0
 25                  and id = prior id
 26                  and prior dbms_random.value is not null;
 27
 28        insert into groups_jobs
 29           select g.groupkey
 30                , regexp_substr(str, '[^,]+', 1, level) job
 31             from (
 32                    select rownum id
 33                         , g.jobs str
 34                      from dual
 35                  )
 36           connect by instr(str, ',', 1, level-1) > 0
 37                  and id = prior id
 38                  and prior dbms_random.value is not null;
 39
 40     end loop;
 41  end;
 42  /

PL/SQL procedure successfully completed.

The tables now with these data:

SQL> select *
  2    from groups
  3   order by groupkey
  4  /

  GROUPKEY
----------
         1
         2
         3
         4

SQL> select *
  2    from groups_depts
  3   order by groupkey, deptno
  4  /

  GROUPKEY     DEPTNO
---------- ----------
         1         10
         2         10
         2         20
         2         30
         3         20
         4         30

6 rows selected.

SQL> select *
  2    from groups_jobs
  3   order by groupkey, job
  4  /

  GROUPKEY JOB
---------- ---------
         1 PRESIDENT
         2 CLERK
         2 MANAGER
         3 ANALYST
         4 SALESMAN

I can now these data the same result as before (just to test, I created the desired data):

SQL> select g.groupkey
  2       , d.deptnos
  3       , j.jobs
  4    from groups g
  5    join (
  6           select groupkey
  7                , rtrim(xmlagg(xmlelement(d,deptno,',').extract('//text()') order by deptno),',') deptnos
  8             from groups_depts
  9            group by groupkey
 10         ) d
 11         on d.groupkey = g.groupkey
 12    join (
 13           select groupkey
 14                , rtrim(xmlagg(xmlelement(j,job,',').extract('//text()') order by job),',') jobs
 15             from groups_jobs
 16            group by groupkey
 17         ) j
 18         on j.groupkey = g.groupkey
 19  /

  GROUPKEY DEPTNOS                        JOBS
---------- ------------------------------ ------------------------------
         1 10                             PRESIDENT
         2 10,20,30                       CLERK,MANAGER
         3 20                             ANALYST
         4 30                             SALESMAN

So far so good. It works all the pretty much as desired - with the exception of a couple things:

The insertion code very simple loop will be slow. OK, it's a work of unique conversion (in theory, but very few times at least), so that could probably be acceptable (except for my professional pride ;-).)

But it's worse, I have groups where the aggregation of string does not work - the chain should be about varchar2 (10000) which does not work in SQL in the group by :-(.

So I tried an attempt from the collections. First a collection of deptnos:

SQL> create type deptno_tab_type as table of number(2)
  2  /

Type created.

SQL> select od.job
  2       , cast(collect(od.deptno order by od.deptno) as deptno_tab_type) deptnos
  3    from orig_data od
  4   group by od.job
  5  /

JOB       DEPTNOS
--------- ------------------------------
ANALYST   DEPTNO_TAB_TYPE(20)
CLERK     DEPTNO_TAB_TYPE(10, 20, 30)
MANAGER   DEPTNO_TAB_TYPE(10, 20, 30)
PRESIDENT DEPTNO_TAB_TYPE(10)
SALESMAN  DEPTNO_TAB_TYPE(30)

All very good - no problem here. But then a collection of jobs:

SQL> create type job_tab_type as table of varchar2(9)
  2  /

Type created.

SQL> select s2.deptnos
  2       , cast(collect(s2.job order by s2.job) as job_tab_type) jobs
  3    from (
  4     select od.job
  5          , cast(collect(od.deptno order by od.deptno) as deptno_tab_type) deptnos
  6       from orig_data od
  7      group by od.job
  8         ) s2
  9   group by s2.deptnos
 10  /
 group by s2.deptnos
          *
ERROR at line 9:
ORA-00932: inkonsistente datatyper: forventede -, fik XAL_SUPERVISOR.DEPTNO_TAB_TYPE

Now, it fails - I can't group by a collection data type...

I'm not asking anyone to write my code, but I know that there are sharper brains out there on the forums of ;-).

Someone would have an idea of something I could try that will allow me to create these "groups of groups" even for large aggregation of chain that technical groups can manage?

Thanks for any help, advice or tips ;-)

The issue of "group-by-collection" can be resolved by creating a container object on which we define a method of CONTROL:

SQL> create type deptno_container as object (
  2    nt deptno_tab_type
  3  , order member function match (o deptno_container) return integer
  4  );
  5  /

Type created

SQL> create or replace type body deptno_container as
  2    order member function match (o deptno_container) return integer is
  3    begin
  4      return case when nt = o.nt then 0 else 1 end;
  5    end;
  6  end;
  7  /

Type body created

Then an INSERT statement can do the job, after unnesting collections:

SQL> insert all
  2    when rn0 = 1 then into groups (groupkey) values (gid)
  3    when rn1 = 1 then into groups_jobs (groupkey, job) values(gid, job)
  4    when rn2 = 1 then into groups_depts (groupkey, deptno) values(gid, deptno)
  5  with all_groups as (
  6    select s2.deptnos
  7         , cast(collect(s2.job order by s2.job) as job_tab_type) jobs
  8         , row_number() over(order by null) gid
  9    from (
 10      select od.job
 11           , deptno_container(
 12               cast(collect(od.deptno order by od.deptno) as deptno_tab_type)
 13             ) deptnos
 14      from orig_data od
 15      group by od.job
 16    ) s2
 17    group by s2.deptnos
 18  )
 19  select gid
 20       , value(j) job
 21       , value(d) deptno
 22       , row_number() over(partition by gid order by null) rn0
 23       , row_number() over(partition by gid, value(j) order by null) rn1
 24       , row_number() over(partition by gid, value(d) order by null) rn2
 25  from all_groups t
 26     , table(t.jobs) j
 27     , table(t.deptnos.nt) d
 28  ;

15 rows inserted

SQL> select * from groups;

  GROUPKEY
----------
         1
         2
         3
         4

SQL> select * from groups_jobs;

  GROUPKEY JOB
---------- ---------
         1 SALESMAN
         2 PRESIDENT
         3 CLERK
         3 MANAGER
         4 ANALYST

SQL> select * from groups_depts;

  GROUPKEY DEPTNO
---------- ------
         1     30
         2     10
         3     10
         3     30
         3     20
         4     20

6 rows selected

Works very well on the sample data, but how this approach is changing on much (much) more large data set is another story :)

Tags: Database

Group by groups

Similar Questions

Maybe you are looking for