TileDB R Notebook

Prelimaries

We will the two basic examples for a dense and sparse array simply to create array data on disk to refer to later in examples that follow.

create_array(uridense)

[1] "/tmp/RtmpaoIiNJ/dense"

API Usage

Creating Arrays

Create Dimension

library(tiledb)

# Create dimension
# In C++: Dimension::create<T>(...)
dim <- tiledb_dim("dim", c(1L, 4L), 2L, "INT32")

Create Array Domain

library(tiledb)

#  .. create dimensions `dim1`, `dim2`
dim1 <- tiledb_dim("dim1", c(1L, 4L), 2L, "INT32")
dim2 <- tiledb_dim("dim2", c(1L, 2L), 2L, "INT32")

# Create domain with two dimensions
# In C++: domain.add_dimensions(dim1).add_dimension(dim2)
dom <- tiledb_domain(dims = c(dim1, dim2))

Creating Attributes

# Create attribute
# In C++: Attribute::create<T>(ctx, "attr")
attr <- tiledb_attr("attr", "INT32")

# We can also set the number of attributes to a value other than one
tiledb:::libtiledb_attribute_set_cell_val_num(attr@ptr, 3)

# And we can set the a variable number of attributes (using NA to select variable size)
tiledb:::libtiledb_attribute_set_cell_val_num(attr@ptr, NA)
## TODO: R function for level setter

Setting a Compressor

comp <- tiledb_filter("GZIP")
tiledb_filter_set_option(comp,"COMPRESSION_LEVEL", 10)

# Create a filter list with the compressor
filter_list <- tiledb_filter_list(c(comp))

# Create attribute with the filter list
attr <- tiledb_attr("attr", "INT32", filter_list = filter_list)

Setting Other Filters

# Create filters
f1 <- tiledb_filter("BIT_WIDTH_REDUCTION")
f2 <- tiledb_filter("ZSTD")

# Create a filter list with the two filters
filter_list <- tiledb_filter_list(c(f1,f2))

# Create attribute with the filter list
attr <- tiledb_attr("attr", "INT32", filter_list = filter_list)

Setting the Tile Chunk Size

# ... create filter list
set_max_chunk_size(filter_list, 10000)

Creating the Array Schema

# ... create domain dom
attr1 <- tiledb_attr("attr1", "INT32", filter_list = filter_list)
attr2 <- tiledb_attr("attr2", "FLOAT64", filter_list = filter_list)

# Create a dense array
schema <- tiledb_array_schema(dom, c(attr1, attr2), sparse = FALSE)
# Or, create a spaese array
# schema <- tiledb_array_schema(dom, c(attr1, attr2), sparse = TRUE)

Setting the Tile and Cell Order

# ... create domain dom
# ... create attributes attr1, attr2

# The tile and order can be "ROW_MAJOR" or "COL_MAJOR"
schema <- tiledb_array_schema(dom, c(attr1, attr2),
                              cell_order = "COL_MAJOR",
                              tile_order = "ROW_MAJOR")

Setting the Data Tile Capacity

tiledb:::libtiledb_array_schema_set_capacity(schema@ptr, 100000)
tiledb:::libtiledb_array_schema_get_capacity(schema@ptr)

[1] 100000

## TODO R function for accessor

Setting Coordinate and Offset Filters

# ... create domain dom
# ... create attributes attr1, attr2
# ... create filter lists fl1, fl2, similar to attributes
f1 <- tiledb_filter("BIT_WIDTH_REDUCTION")
f2 <- tiledb_filter("ZSTD")
fl1 <- tiledb_filter_list(c(f1))
fl2 <- tiledb_filter_list(c(f2))

# Create the schema setting the coordinates and offsets filter lists
schema <- tiledb_array_schema(dom, c(attr1, attr2),
                              coords_filter_list = fl1,
                              offsets_filter_list = fl2)

Checking Correctness

tiledb:::libtiledb_array_schema_check(schema@ptr)
## TODO: R function for access

Creating the Array

# ... create array schema

# Create the array
tiledb_array_create(uridense, schema)

Creating Encrypted Arrays

encryption_key <- "0123456789abcdeF0123456789abcdeF"
uridensewkey <- file.path(tdir, "densewkey")
if (dir.exists(uridensewkey)) unlink(uridensewkey, recursive=TRUE)

# shortcut: borrow schema from dense array; could create schema too
schema <- schema(tiledb_dense(uridense))
tiledb:::libtiledb_array_create_with_key(uridensewkey, schema@ptr, encryption_key)

[1] "/tmp/RtmppsnBrT/densewkey"

##TODO: R support

Writing Arrays

Writing in Dense Subarrays

## prepare a larger 5 x 5 to embed into
tmp <- tempfile()
d1  <- tiledb_dim(domain = c(1L, 5L))
d2  <- tiledb_dim(domain = c(1L, 5L))
dom <- tiledb_domain(c(d1, d2))
val <- tiledb_attr(name="val", type = "INT32")
sch <- tiledb_array_schema(dom, c(val))
tiledb_array_create(tmp, sch)

dat <- matrix(as.integer(rnorm(25)*100), 5, 5)
arr <- tiledb_dense(tmp, as.data.frame=FALSE)
arr[] <- dat


# Prepare a 2x3 dense array
# Contrary to Python, R by default stores arrays in col-major order
data <-  array(c(1L, 4L, 2L, 5L, 3L, 6L), dim=c(2,3))

# Prepare the [1,2] x [2,4] subarray to write to
I <- c(1:2)
J <- c(2:4)

# Open the array and write the data to it
A <- tiledb_dense(uri = tmp)
A[I, J] <- data

unlink(tmp, recursive=TRUE)

Basic Writing using Low-Level Code

ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridense, "WRITE")

## data: simple (integer sequence) of 1:16 times 10
vec <- 1:16 * 10L
subarr <- c(1L,4L, 1L,4L)

qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "WRITE")
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", vec)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
res <- tiledb:::libtiledb_array_close(arrptr)

Writing Sparse Cells

tmp <- urisparse
unlink(tmp, recursive=TRUE)

d1  <- tiledb_dim(domain = c(1L, 5L))
d2  <- tiledb_dim(domain = c(1L, 5L))
dom <- tiledb_domain(c(d1, d2))
val <- tiledb_attr("val", type = "INT32")
sch <- tiledb_array_schema(dom, c(val), sparse=TRUE)
tiledb_array_create(tmp, sch)

[1] "/tmp/RtmppsnBrT/sparse"

# Prepare some data
data <- c(3L, 4L, 1L, 2L)

I <- c(3, 4, 1, 2)
J <- c(3, 4, 2, 1)

# Open the array and write the data to it
A <- tiledb_sparse(uri = tmp)
A[I, J] <- data

Writing Encrypted Arrays

ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open_with_key(ctx@ptr, uridensewkey, "WRITE", encryption_key)

## data: simple (integer sequence) of 1:16 times 10
vec <- 1:16 * 10L
subarr <- c(1L,4L, 1L,4L)

qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "WRITE")
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", vec)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
res <- tiledb:::libtiledb_array_close(arrptr)
# TODO Higher-level R support

Fixed-length Attributes

if (dir.exists(uridensefix)) unlink(uridensefix, recursive=TRUE)
d1  <- tiledb_dim(domain = c(1L, 4L))
d2  <- tiledb_dim(domain = c(1L, 4L))
dom <- tiledb_domain(c(d1, d2))

vec <- 1:32 * 10L
attr <- tiledb_attr("a", type = r_to_tiledb_type(vec))

## set to two values per cell
tiledb:::libtiledb_attribute_set_cell_val_num(attr@ptr, 2)
sch <- tiledb_array_schema(dom, c(attr))
tiledb_array_create(uridensefix, sch)

[1] "/tmp/RtmppsnBrT/densefix"

ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridensefix, "WRITE")
subarr <- c(1L,4L, 1L,4L)

qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "WRITE")
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", vec)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
res <- tiledb:::libtiledb_array_close(arrptr)

#TODO Higher-level R support

Var-length Attributes

if (dir.exists(uridensevar)) unlink(uridensevar, recursive=TRUE)
## Define array
## The array will be 4x4 with dimensions "rows" and "cols", with domain [1,4].
dom <- tiledb_domain(dims = c(tiledb_dim("rows", c(1L, 4L), 4L, "INT32"),
                              tiledb_dim("cols", c(1L, 4L), 4L, "INT32")))


attr <- tiledb_attr("a1", type = "CHAR")
## set to variable length
tiledb:::libtiledb_attribute_set_cell_val_num(attr@ptr, NA)

## now set the schema
ctx <- tiledb_ctx()
schptr <- tiledb:::libtiledb_array_schema_create(ctx@ptr, "DENSE")
tiledb:::libtiledb_array_schema_set_domain(schptr, dom@ptr)
tiledb:::libtiledb_array_schema_set_cell_order(schptr, "ROW_MAJOR")
tiledb:::libtiledb_array_schema_set_tile_order(schptr, "ROW_MAJOR")
tiledb:::libtiledb_array_schema_add_attribute(schptr, attr@ptr)



## Create the (empty) array on disk.
tiledb:::libtiledb_array_create(uridensevar, schptr)

[1] "/tmp/RtmppsnBrT/densevar"

data <- "abbcccddeeefghhhijjjkklmnoop";
offsets <- c(0L, 1L, 3L, 6L, 8L, 11L, 12L, 13L, 16L, 17L, 20L, 22L, 23L, 24L, 25L, 27L)

ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridensevar, "WRITE")
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "WRITE")
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")

bufptr <- tiledb:::libtiledb_query_buffer_var_char_create(offsets, data)
qryptr <- tiledb:::libtiledb_query_set_buffer_var_char(qryptr, "a1", bufptr)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
tiledb:::libtiledb_array_close(arrptr)

<pointer: 0x7fb5fc018dc0>

#TODO Higher-level R support

Getting the Fragment Info

## continuing from previous example on dense variable length array
## (but this works of course with any array after a write is needed
numfrag <- tiledb:::libtiledb_query_get_fragment_num(qryptr)
uritxt <- tiledb:::libtiledb_query_get_fragment_uri(qryptr, 0)

##TODO Higher-level R support

Reading Arrays

Reading the Array Schema

array_name <- urisparse
A <- tiledb_sparse(uri = array_name)
# can also load dense
sch <- schema(A)
# can also open encrypted arrays with key

Inspecting the array schema

array_name <- urisparse

# Get array schema
A <- tiledb_sparse(uri = array_name)
# or use tiledb_dense(uri = array_name) for dense
schema <- schema(A)

# Get array type
sparse <- is.sparse(schema)

# Get tile capacity
tiledb:::libtiledb_array_schema_get_capacity(sch@ptr)

[1] 10000

# TODO: R accessor for tile

# Get tile order
t_order <- tile_order(schema)

# Get cell order
c_order <- cell_order(schema)

# Get coordinates and offset filter list
reslist <- filter_list(schema)

# Get the array domain
dom <- domain(schema)

# Get all attributes as list
attrs <- attrs(schema)

# Get attribute from name
attr <- attrs(schema, "attr")

# Dump the array schema in ASCII format to stdout
show(schema)

- Array type: sparse
- Cell order: col-major
- Tile order: col-major
- Capacity: 10000
- Coordinates compressor: ZSTD
- Coordinates compression level: -1

=== Domain ===
- Dimensions type: INT32

### Dimension ###
- Name: <anonymous>
- Domain: [1,5]
- Tile extent: 5

### Dimension ###
- Name: <anonymous>
- Domain: [1,5]
- Tile extent: 5

### Attribute ###
- Name: val
- Type: INT32
- Compressor: NO_COMPRESSION
- Compression level: -1
- Cell val num: 1

Inspecting Domain

# ... get array schema
# ... get domain from schema
dom <- domain(schema)

# Get the domain datatype (i.e., the datatype of all dimensions)
type <- datatype(dom)

# Get number of dimensions
dim_num <- dim(dom)

# Get all dimension
dims <- dimensions(dom)

# Dump the domain in ASCII format in the selected output
show(dom)

=== Domain ===
- Dimensions type: INT32

### Dimension ###
- Name: <anonymous>
- Domain: [1,5]
- Tile extent: 5

### Dimension ###
- Name: <anonymous>
- Domain: [1,5]
- Tile extent: 5

Inspecting Dimensions

# ... get array schema
# ... get domain
# ... get dimension by index or name

#FIXME

# Get dimension name
dim_name <- name(dim)

# Get dimension datatype
dim_type <- datatype(dim)

# Get dimension domain
domain <- domain(dim)

# Get tile extent
tile_extent <- tile(dim)

# Dump the dimension in ASCII format in the selected output
show(dim)

Inspecting Attributes

# ... get array schema
# ... get attribute by index or name

stopifnot(is.null(attr))
# Get attribute name
attr_name <- name(attr)

# Get attribute datatype
attr_type <- datatype(attr)

# Get filter list
filter_list <- filter_list(attr)

# Check if attribute is variable-length
# variable length attribtues are not supported in R currently

# Get number of values per cell
num <- ncells(attr)

# Get cell size for this attribute
sz <-- tiledb:::libtiledb_attribute_get_cell_val(attr@ptr)

# Dump the attribute in ASCII format in the selected output
show(attr)

Inspecting Filters

fl <- tiledb_filter_list()

# get number of filter
tiledb:::libtiledb_filter_list_get_nfilters(fl@ptr)

# get max chunk size
tiledb:::libtiledb_filter_list_get_max_chunk_size(fl@ptr)

# get filter from index
ind <- i
tiledb:::libtiledb_filter_list_get_filter_from_index(fl@ptr, ind)

Basic Reading

# Create a TileDB context
ctx <- tiledb_ctx()

# Open a dense array
A <- tiledb_dense(uri = uridense, ctx=ctx)

# Or, open a sparse array
# A <- tiledb_sparse(uri = "<array-uri>", ctx=ctx)

# Slice only rows 1, 2 and cols 2, 3, 4
data <- A[1:2, 2:4]
show(data)

     [,1] [,2] [,3]
[1,]   20   30   40
[2,]   60   70   80

Basic Reading using Low-Level Code

ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridense, "READ")
## subarray of rows 1,2 and cols 2,3,4
subarr <- c(1L,2L, 2L,4L)

qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
v <- integer(6)  # reserve space
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", v)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
print(v)         # unformed array, no coordinates

[1] 20 30 40 60 70 80

res <- tiledb:::libtiledb_array_close(arrptr)

Reading Encrypted Arrays

ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open_with_key(ctx@ptr, uridensewkey, "READ", encryption_key)

## subarray of rows 1,2 and cols 2,3,4
subarr <- c(1L,2L, 2L,4L)

qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
v <- integer(6)  # reserve space
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", v)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
print(v)         # unformed array, no coordinates

[1] 20 30 40 60 70 80

res <- tiledb:::libtiledb_array_close(arrptr)

## TODO open at timestamp example

Multi-range Subarrays

ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridense, "READ")

qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")

## range of rows 1 and 2, and 4 for dim 1, all rows for dim 2
qryptr <- tiledb:::libtiledb_query_add_range(qryptr, 0, 1L, 2L)
qryptr <- tiledb:::libtiledb_query_add_range(qryptr, 0, 4L, 4L)
qryptr <- tiledb:::libtiledb_query_add_range(qryptr, 1, 1L, 4L)

qryptr <- tiledb:::libtiledb_query_set_layout(qryptr, "ROW_MAJOR")
v <- integer(12)  # reserve space
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", v)
qryptr <- tiledb:::libtiledb_query_submit(qryptr)
print(v)         # unformed array, no coordinates

 [1]  10  20  30  40  50  60  70  80 130 140 150 160

res <- tiledb:::libtiledb_array_close(arrptr)

Incomplete Queries

ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridense, "READ")
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")
subarr <- c(1L,4L, 1L,4L)
qryptr <- tiledb:::libtiledb_query_set_subarray(qryptr, subarr)
vec <- integer(4)  # reserve (insufficient) space
qryptr <- tiledb:::libtiledb_query_set_buffer(qryptr, "a", vec)
finished <- FALSE
while (!finished) {
  qryptr <- tiledb:::libtiledb_query_submit(qryptr)
  print(vec)
  finished <- tiledb:::libtiledb_query_status(qryptr) == "COMPLETE"
}

[1] 10 20 30 40
[1] 50 60 70 80
[1]  90 100 110 120
[1] 130 140 150 160

res <- tiledb:::libtiledb_array_close(arrptr)

Result Estimation

ctx <- tiledb_ctx()
arrptr <- tiledb:::libtiledb_array_open(ctx@ptr, uridensevar, "READ")
qryptr <- tiledb:::libtiledb_query(ctx@ptr, arrptr, "READ")
estsz <- tiledb:::libtiledb_query_get_est_result_size_var(qryptr, "a1")
res <- tiledb:::libtiledb_array_close(arrptr)
estsz

[1] 16 28

Time Traveling

v

[1] 2 3 4 6 7 8

Embedded SQL

Asynchronous Queries

Configuration

Array Metadata

Writing Array Metadata

res <- tiledb_put_metadata(uridense, "aaa", 100L)
res <- tiledb_put_metadata(uridense, "bb", c(1.1, 2.2))

Reading Array Metadata

One can read by key:

tiledb_get_metadata(uridense, "aaa")

Or one can retrieve all metadata at once:

md <- tiledb_get_all_metadata(uridense)
print(md)

Deleting Array Metadata

res <- tiledb_delete_metadata(uridense, "aaa")