Tensorflow: RaggedTensor.from_tensor flattening values from all arrays into one array, instead of preserving...

Multi tool use
In the official documentation, RaggedTensor.from_tensor
will work something like this.
x = [[1, 3, -1, -1], [2, -1, -1, -1], [4, 5, 8, 9]]
print(tf.RaggedTensor.from_tensor(x, padding=-1))
Output:
<tf.RaggedTensor [[1, 3], [2], [4, 5, 8, 9]]>
Preserving the original number of arrays.
However, when working with a batch outputted by the dataset api iterator, it flattens it to one array. Here is the key parts of the code.
dataset = dataset.padded_batch(3, padded_shapes=([None],[None]), padding_values=(tf.constant(-1, dtype=tf.int64)
,tf.constant(-1, dtype=tf.int64)))
iterator = dataset.make_one_shot_iterator()
i, data = iterator.get_next()
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
with tf.Session() as sess:
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
Here is the output
[array([[ 0, 1, 2, 3, -1],
[ 2, 3, 4, -1, -1],
[ 3, 6, 5, 4, 3]]), tf.RaggedTensorValue(values=array([0, 1, 2, 3, 2, 3, 4, 3, 6, 5, 4, 3]), row_splits=array([ 0, 4, 7, 12]))]
[array([[ 3, 9, -1, -1],
[ 0, 1, 2, 3],
[ 2, 3, 4, -1]]), tf.RaggedTensorValue(values=array([3, 9, 0, 1, 2, 3, 2, 3, 4]), row_splits=array([0, 2, 6, 9]))]
[array([[ 3, 6, 5, 4, 3],
[ 3, 9, -1, -1, -1],
[ 0, 1, 2, 3, -1]]), tf.RaggedTensorValue(values=array([3, 6, 5, 4, 3, 3, 9, 0, 1, 2, 3]), row_splits=array([ 0, 5, 7, 11]))]
Here is the full code to the minimal example to reproduce the results
!pip install -q tf-nightly
import math
import numpy as np
import tensorflow as tf
#Generate Test data
cells = np.array([[0,1,2,3], [2,3,4], [3,6,5,4,3], [3,9]])
mells = np.array([[0], [2], [3], [9]])
print(cells)
#Write test data to tf.records file
writer = tf.python_io.TFRecordWriter('test.tfrecords')
for index in range(mells.shape[0]):
example = tf.train.Example(features=tf.train.Features(feature={
'num_value':tf.train.Feature(int64_list=tf.train.Int64List(value=mells[index])),
'list_value':tf.train.Feature(int64_list=tf.train.Int64List(value=cells[index]))
}))
writer.write(example.SerializeToString())
writer.close()
#Open tfrecords file and generate batch from data
filenames = ["test.tfrecords"]
dataset = tf.data.TFRecordDataset(filenames)
def _parse_function(example_proto):
keys_to_features = {'num_value':tf.VarLenFeature(tf.int64),
'list_value':tf.VarLenFeature(tf.int64)}
parsed_features = tf.parse_single_example(example_proto, keys_to_features)
return tf.sparse.to_dense(parsed_features['num_value']),
tf.sparse.to_dense(parsed_features['list_value'])
# Parse the record into tensors.
dataset = dataset.map(_parse_function)
# Shuffle the dataset
dataset = dataset.shuffle(buffer_size=1)
# Repeat the input indefinitly
dataset = dataset.repeat()
# Generate batches
dataset = dataset.padded_batch(3, padded_shapes=([None],[None]), padding_values=(tf.constant(-1, dtype=tf.int64)
,tf.constant(-1, dtype=tf.int64)))
iterator = dataset.make_one_shot_iterator()
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
#Print data
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
Here is the official Tensorflow Guide to ragged tensors
https://www.tensorflow.org/guide/ragged_tensors
And official Tensorflow documentation
https://www.tensorflow.org/versions/r1.13/api_docs/python/tf/RaggedTensor
python tensorflow tensorflow-datasets
add a comment |
In the official documentation, RaggedTensor.from_tensor
will work something like this.
x = [[1, 3, -1, -1], [2, -1, -1, -1], [4, 5, 8, 9]]
print(tf.RaggedTensor.from_tensor(x, padding=-1))
Output:
<tf.RaggedTensor [[1, 3], [2], [4, 5, 8, 9]]>
Preserving the original number of arrays.
However, when working with a batch outputted by the dataset api iterator, it flattens it to one array. Here is the key parts of the code.
dataset = dataset.padded_batch(3, padded_shapes=([None],[None]), padding_values=(tf.constant(-1, dtype=tf.int64)
,tf.constant(-1, dtype=tf.int64)))
iterator = dataset.make_one_shot_iterator()
i, data = iterator.get_next()
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
with tf.Session() as sess:
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
Here is the output
[array([[ 0, 1, 2, 3, -1],
[ 2, 3, 4, -1, -1],
[ 3, 6, 5, 4, 3]]), tf.RaggedTensorValue(values=array([0, 1, 2, 3, 2, 3, 4, 3, 6, 5, 4, 3]), row_splits=array([ 0, 4, 7, 12]))]
[array([[ 3, 9, -1, -1],
[ 0, 1, 2, 3],
[ 2, 3, 4, -1]]), tf.RaggedTensorValue(values=array([3, 9, 0, 1, 2, 3, 2, 3, 4]), row_splits=array([0, 2, 6, 9]))]
[array([[ 3, 6, 5, 4, 3],
[ 3, 9, -1, -1, -1],
[ 0, 1, 2, 3, -1]]), tf.RaggedTensorValue(values=array([3, 6, 5, 4, 3, 3, 9, 0, 1, 2, 3]), row_splits=array([ 0, 5, 7, 11]))]
Here is the full code to the minimal example to reproduce the results
!pip install -q tf-nightly
import math
import numpy as np
import tensorflow as tf
#Generate Test data
cells = np.array([[0,1,2,3], [2,3,4], [3,6,5,4,3], [3,9]])
mells = np.array([[0], [2], [3], [9]])
print(cells)
#Write test data to tf.records file
writer = tf.python_io.TFRecordWriter('test.tfrecords')
for index in range(mells.shape[0]):
example = tf.train.Example(features=tf.train.Features(feature={
'num_value':tf.train.Feature(int64_list=tf.train.Int64List(value=mells[index])),
'list_value':tf.train.Feature(int64_list=tf.train.Int64List(value=cells[index]))
}))
writer.write(example.SerializeToString())
writer.close()
#Open tfrecords file and generate batch from data
filenames = ["test.tfrecords"]
dataset = tf.data.TFRecordDataset(filenames)
def _parse_function(example_proto):
keys_to_features = {'num_value':tf.VarLenFeature(tf.int64),
'list_value':tf.VarLenFeature(tf.int64)}
parsed_features = tf.parse_single_example(example_proto, keys_to_features)
return tf.sparse.to_dense(parsed_features['num_value']),
tf.sparse.to_dense(parsed_features['list_value'])
# Parse the record into tensors.
dataset = dataset.map(_parse_function)
# Shuffle the dataset
dataset = dataset.shuffle(buffer_size=1)
# Repeat the input indefinitly
dataset = dataset.repeat()
# Generate batches
dataset = dataset.padded_batch(3, padded_shapes=([None],[None]), padding_values=(tf.constant(-1, dtype=tf.int64)
,tf.constant(-1, dtype=tf.int64)))
iterator = dataset.make_one_shot_iterator()
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
#Print data
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
Here is the official Tensorflow Guide to ragged tensors
https://www.tensorflow.org/guide/ragged_tensors
And official Tensorflow documentation
https://www.tensorflow.org/versions/r1.13/api_docs/python/tf/RaggedTensor
python tensorflow tensorflow-datasets
add a comment |
In the official documentation, RaggedTensor.from_tensor
will work something like this.
x = [[1, 3, -1, -1], [2, -1, -1, -1], [4, 5, 8, 9]]
print(tf.RaggedTensor.from_tensor(x, padding=-1))
Output:
<tf.RaggedTensor [[1, 3], [2], [4, 5, 8, 9]]>
Preserving the original number of arrays.
However, when working with a batch outputted by the dataset api iterator, it flattens it to one array. Here is the key parts of the code.
dataset = dataset.padded_batch(3, padded_shapes=([None],[None]), padding_values=(tf.constant(-1, dtype=tf.int64)
,tf.constant(-1, dtype=tf.int64)))
iterator = dataset.make_one_shot_iterator()
i, data = iterator.get_next()
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
with tf.Session() as sess:
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
Here is the output
[array([[ 0, 1, 2, 3, -1],
[ 2, 3, 4, -1, -1],
[ 3, 6, 5, 4, 3]]), tf.RaggedTensorValue(values=array([0, 1, 2, 3, 2, 3, 4, 3, 6, 5, 4, 3]), row_splits=array([ 0, 4, 7, 12]))]
[array([[ 3, 9, -1, -1],
[ 0, 1, 2, 3],
[ 2, 3, 4, -1]]), tf.RaggedTensorValue(values=array([3, 9, 0, 1, 2, 3, 2, 3, 4]), row_splits=array([0, 2, 6, 9]))]
[array([[ 3, 6, 5, 4, 3],
[ 3, 9, -1, -1, -1],
[ 0, 1, 2, 3, -1]]), tf.RaggedTensorValue(values=array([3, 6, 5, 4, 3, 3, 9, 0, 1, 2, 3]), row_splits=array([ 0, 5, 7, 11]))]
Here is the full code to the minimal example to reproduce the results
!pip install -q tf-nightly
import math
import numpy as np
import tensorflow as tf
#Generate Test data
cells = np.array([[0,1,2,3], [2,3,4], [3,6,5,4,3], [3,9]])
mells = np.array([[0], [2], [3], [9]])
print(cells)
#Write test data to tf.records file
writer = tf.python_io.TFRecordWriter('test.tfrecords')
for index in range(mells.shape[0]):
example = tf.train.Example(features=tf.train.Features(feature={
'num_value':tf.train.Feature(int64_list=tf.train.Int64List(value=mells[index])),
'list_value':tf.train.Feature(int64_list=tf.train.Int64List(value=cells[index]))
}))
writer.write(example.SerializeToString())
writer.close()
#Open tfrecords file and generate batch from data
filenames = ["test.tfrecords"]
dataset = tf.data.TFRecordDataset(filenames)
def _parse_function(example_proto):
keys_to_features = {'num_value':tf.VarLenFeature(tf.int64),
'list_value':tf.VarLenFeature(tf.int64)}
parsed_features = tf.parse_single_example(example_proto, keys_to_features)
return tf.sparse.to_dense(parsed_features['num_value']),
tf.sparse.to_dense(parsed_features['list_value'])
# Parse the record into tensors.
dataset = dataset.map(_parse_function)
# Shuffle the dataset
dataset = dataset.shuffle(buffer_size=1)
# Repeat the input indefinitly
dataset = dataset.repeat()
# Generate batches
dataset = dataset.padded_batch(3, padded_shapes=([None],[None]), padding_values=(tf.constant(-1, dtype=tf.int64)
,tf.constant(-1, dtype=tf.int64)))
iterator = dataset.make_one_shot_iterator()
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
#Print data
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
Here is the official Tensorflow Guide to ragged tensors
https://www.tensorflow.org/guide/ragged_tensors
And official Tensorflow documentation
https://www.tensorflow.org/versions/r1.13/api_docs/python/tf/RaggedTensor
python tensorflow tensorflow-datasets
In the official documentation, RaggedTensor.from_tensor
will work something like this.
x = [[1, 3, -1, -1], [2, -1, -1, -1], [4, 5, 8, 9]]
print(tf.RaggedTensor.from_tensor(x, padding=-1))
Output:
<tf.RaggedTensor [[1, 3], [2], [4, 5, 8, 9]]>
Preserving the original number of arrays.
However, when working with a batch outputted by the dataset api iterator, it flattens it to one array. Here is the key parts of the code.
dataset = dataset.padded_batch(3, padded_shapes=([None],[None]), padding_values=(tf.constant(-1, dtype=tf.int64)
,tf.constant(-1, dtype=tf.int64)))
iterator = dataset.make_one_shot_iterator()
i, data = iterator.get_next()
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
with tf.Session() as sess:
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
Here is the output
[array([[ 0, 1, 2, 3, -1],
[ 2, 3, 4, -1, -1],
[ 3, 6, 5, 4, 3]]), tf.RaggedTensorValue(values=array([0, 1, 2, 3, 2, 3, 4, 3, 6, 5, 4, 3]), row_splits=array([ 0, 4, 7, 12]))]
[array([[ 3, 9, -1, -1],
[ 0, 1, 2, 3],
[ 2, 3, 4, -1]]), tf.RaggedTensorValue(values=array([3, 9, 0, 1, 2, 3, 2, 3, 4]), row_splits=array([0, 2, 6, 9]))]
[array([[ 3, 6, 5, 4, 3],
[ 3, 9, -1, -1, -1],
[ 0, 1, 2, 3, -1]]), tf.RaggedTensorValue(values=array([3, 6, 5, 4, 3, 3, 9, 0, 1, 2, 3]), row_splits=array([ 0, 5, 7, 11]))]
Here is the full code to the minimal example to reproduce the results
!pip install -q tf-nightly
import math
import numpy as np
import tensorflow as tf
#Generate Test data
cells = np.array([[0,1,2,3], [2,3,4], [3,6,5,4,3], [3,9]])
mells = np.array([[0], [2], [3], [9]])
print(cells)
#Write test data to tf.records file
writer = tf.python_io.TFRecordWriter('test.tfrecords')
for index in range(mells.shape[0]):
example = tf.train.Example(features=tf.train.Features(feature={
'num_value':tf.train.Feature(int64_list=tf.train.Int64List(value=mells[index])),
'list_value':tf.train.Feature(int64_list=tf.train.Int64List(value=cells[index]))
}))
writer.write(example.SerializeToString())
writer.close()
#Open tfrecords file and generate batch from data
filenames = ["test.tfrecords"]
dataset = tf.data.TFRecordDataset(filenames)
def _parse_function(example_proto):
keys_to_features = {'num_value':tf.VarLenFeature(tf.int64),
'list_value':tf.VarLenFeature(tf.int64)}
parsed_features = tf.parse_single_example(example_proto, keys_to_features)
return tf.sparse.to_dense(parsed_features['num_value']),
tf.sparse.to_dense(parsed_features['list_value'])
# Parse the record into tensors.
dataset = dataset.map(_parse_function)
# Shuffle the dataset
dataset = dataset.shuffle(buffer_size=1)
# Repeat the input indefinitly
dataset = dataset.repeat()
# Generate batches
dataset = dataset.padded_batch(3, padded_shapes=([None],[None]), padding_values=(tf.constant(-1, dtype=tf.int64)
,tf.constant(-1, dtype=tf.int64)))
iterator = dataset.make_one_shot_iterator()
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
#Print data
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
print(sess.run([ data, data2 ]))
Here is the official Tensorflow Guide to ragged tensors
https://www.tensorflow.org/guide/ragged_tensors
And official Tensorflow documentation
https://www.tensorflow.org/versions/r1.13/api_docs/python/tf/RaggedTensor
python tensorflow tensorflow-datasets
python tensorflow tensorflow-datasets
asked Dec 29 '18 at 22:07
SantoshGupta7SantoshGupta7
6431515
6431515
add a comment |
add a comment |
2 Answers
2
active
oldest
votes
As you discovered, the RaggedTensor
s are not actually getting flattened. Internally, a 2D RaggedTensor
is encoded using two Tensors/arrays: one containing a flat list of values, and the other containing row splits. For more details about how RaggedTensor
s are encoded using underlying tensors/arrays, see: https://www.tensorflow.org/guide/ragged_tensors#raggedtensor_encoding
The confusion was probably coming from the way that RaggedTensors get displayed when printing. Python has two string conversion methods: __str__
and __repr__
. __str__
gets used if you just print a value by itself, and __repr__
gets used if that value is embedded in some larger structure (such as a list).
For RaggedTensorValue, the __str__
method returns "<tf.RaggedTensorValue %s>" % self.to_list()
. I.e., it will show you the value formatted as a list. But the __repr__
method returns "tf.RaggedTensorValue(values=%r, row_splits=%r)" % (self._values, self._row_splits)
. I.e., it will show you the underlying numpy arrays that are used to encode the RaggedTensorValue.
add a comment |
Turns out it's not flattening it, not exactly sure how it works but it looks like by keeping track of where the row breaks are, and then executes them when evaluated.
tf.RaggedTensorValue(values=array([3, 6, 5, 4, 3, 3, 9, 0, 1, 2, 3]), row_splits=array([ 0, 5, 7, 11]))]
'row_splits' keeps track of where to split the rows.
Here are some results with eager execution.
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
results
<tf.RaggedTensor [[3, 9], [0, 1, 2, 3], [2, 3, 4]]>
<tf.RaggedTensor [[3, 6, 5, 4, 3], [3, 9], [0, 1, 2, 3]]>
<tf.RaggedTensor [[2, 3, 4], [3, 6, 5, 4, 3], [3, 9]]>
<tf.RaggedTensor [[0, 1, 2, 3], [2, 3, 4], [3, 6, 5, 4, 3]]>
add a comment |
Your Answer
StackExchange.ifUsing("editor", function () {
StackExchange.using("externalEditor", function () {
StackExchange.using("snippets", function () {
StackExchange.snippets.init();
});
});
}, "code-snippets");
StackExchange.ready(function() {
var channelOptions = {
tags: "".split(" "),
id: "1"
};
initTagRenderer("".split(" "), "".split(" "), channelOptions);
StackExchange.using("externalEditor", function() {
// Have to fire editor after snippets, if snippets enabled
if (StackExchange.settings.snippets.snippetsEnabled) {
StackExchange.using("snippets", function() {
createEditor();
});
}
else {
createEditor();
}
});
function createEditor() {
StackExchange.prepareEditor({
heartbeatType: 'answer',
autoActivateHeartbeat: false,
convertImagesToLinks: true,
noModals: true,
showLowRepImageUploadWarning: true,
reputationToPostImages: 10,
bindNavPrevention: true,
postfix: "",
imageUploader: {
brandingHtml: "Powered by u003ca class="icon-imgur-white" href="https://imgur.com/"u003eu003c/au003e",
contentPolicyHtml: "User contributions licensed under u003ca href="https://creativecommons.org/licenses/by-sa/3.0/"u003ecc by-sa 3.0 with attribution requiredu003c/au003e u003ca href="https://stackoverflow.com/legal/content-policy"u003e(content policy)u003c/au003e",
allowUrls: true
},
onDemand: true,
discardSelector: ".discard-answer"
,immediatelyShowMarkdownHelp:true
});
}
});
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53973709%2ftensorflow-raggedtensor-from-tensor-flattening-values-from-all-arrays-into-one%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
2 Answers
2
active
oldest
votes
2 Answers
2
active
oldest
votes
active
oldest
votes
active
oldest
votes
As you discovered, the RaggedTensor
s are not actually getting flattened. Internally, a 2D RaggedTensor
is encoded using two Tensors/arrays: one containing a flat list of values, and the other containing row splits. For more details about how RaggedTensor
s are encoded using underlying tensors/arrays, see: https://www.tensorflow.org/guide/ragged_tensors#raggedtensor_encoding
The confusion was probably coming from the way that RaggedTensors get displayed when printing. Python has two string conversion methods: __str__
and __repr__
. __str__
gets used if you just print a value by itself, and __repr__
gets used if that value is embedded in some larger structure (such as a list).
For RaggedTensorValue, the __str__
method returns "<tf.RaggedTensorValue %s>" % self.to_list()
. I.e., it will show you the value formatted as a list. But the __repr__
method returns "tf.RaggedTensorValue(values=%r, row_splits=%r)" % (self._values, self._row_splits)
. I.e., it will show you the underlying numpy arrays that are used to encode the RaggedTensorValue.
add a comment |
As you discovered, the RaggedTensor
s are not actually getting flattened. Internally, a 2D RaggedTensor
is encoded using two Tensors/arrays: one containing a flat list of values, and the other containing row splits. For more details about how RaggedTensor
s are encoded using underlying tensors/arrays, see: https://www.tensorflow.org/guide/ragged_tensors#raggedtensor_encoding
The confusion was probably coming from the way that RaggedTensors get displayed when printing. Python has two string conversion methods: __str__
and __repr__
. __str__
gets used if you just print a value by itself, and __repr__
gets used if that value is embedded in some larger structure (such as a list).
For RaggedTensorValue, the __str__
method returns "<tf.RaggedTensorValue %s>" % self.to_list()
. I.e., it will show you the value formatted as a list. But the __repr__
method returns "tf.RaggedTensorValue(values=%r, row_splits=%r)" % (self._values, self._row_splits)
. I.e., it will show you the underlying numpy arrays that are used to encode the RaggedTensorValue.
add a comment |
As you discovered, the RaggedTensor
s are not actually getting flattened. Internally, a 2D RaggedTensor
is encoded using two Tensors/arrays: one containing a flat list of values, and the other containing row splits. For more details about how RaggedTensor
s are encoded using underlying tensors/arrays, see: https://www.tensorflow.org/guide/ragged_tensors#raggedtensor_encoding
The confusion was probably coming from the way that RaggedTensors get displayed when printing. Python has two string conversion methods: __str__
and __repr__
. __str__
gets used if you just print a value by itself, and __repr__
gets used if that value is embedded in some larger structure (such as a list).
For RaggedTensorValue, the __str__
method returns "<tf.RaggedTensorValue %s>" % self.to_list()
. I.e., it will show you the value formatted as a list. But the __repr__
method returns "tf.RaggedTensorValue(values=%r, row_splits=%r)" % (self._values, self._row_splits)
. I.e., it will show you the underlying numpy arrays that are used to encode the RaggedTensorValue.
As you discovered, the RaggedTensor
s are not actually getting flattened. Internally, a 2D RaggedTensor
is encoded using two Tensors/arrays: one containing a flat list of values, and the other containing row splits. For more details about how RaggedTensor
s are encoded using underlying tensors/arrays, see: https://www.tensorflow.org/guide/ragged_tensors#raggedtensor_encoding
The confusion was probably coming from the way that RaggedTensors get displayed when printing. Python has two string conversion methods: __str__
and __repr__
. __str__
gets used if you just print a value by itself, and __repr__
gets used if that value is embedded in some larger structure (such as a list).
For RaggedTensorValue, the __str__
method returns "<tf.RaggedTensorValue %s>" % self.to_list()
. I.e., it will show you the value formatted as a list. But the __repr__
method returns "tf.RaggedTensorValue(values=%r, row_splits=%r)" % (self._values, self._row_splits)
. I.e., it will show you the underlying numpy arrays that are used to encode the RaggedTensorValue.
answered Jan 11 at 22:13
Edward LoperEdward Loper
9,93833045
9,93833045
add a comment |
add a comment |
Turns out it's not flattening it, not exactly sure how it works but it looks like by keeping track of where the row breaks are, and then executes them when evaluated.
tf.RaggedTensorValue(values=array([3, 6, 5, 4, 3, 3, 9, 0, 1, 2, 3]), row_splits=array([ 0, 5, 7, 11]))]
'row_splits' keeps track of where to split the rows.
Here are some results with eager execution.
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
results
<tf.RaggedTensor [[3, 9], [0, 1, 2, 3], [2, 3, 4]]>
<tf.RaggedTensor [[3, 6, 5, 4, 3], [3, 9], [0, 1, 2, 3]]>
<tf.RaggedTensor [[2, 3, 4], [3, 6, 5, 4, 3], [3, 9]]>
<tf.RaggedTensor [[0, 1, 2, 3], [2, 3, 4], [3, 6, 5, 4, 3]]>
add a comment |
Turns out it's not flattening it, not exactly sure how it works but it looks like by keeping track of where the row breaks are, and then executes them when evaluated.
tf.RaggedTensorValue(values=array([3, 6, 5, 4, 3, 3, 9, 0, 1, 2, 3]), row_splits=array([ 0, 5, 7, 11]))]
'row_splits' keeps track of where to split the rows.
Here are some results with eager execution.
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
results
<tf.RaggedTensor [[3, 9], [0, 1, 2, 3], [2, 3, 4]]>
<tf.RaggedTensor [[3, 6, 5, 4, 3], [3, 9], [0, 1, 2, 3]]>
<tf.RaggedTensor [[2, 3, 4], [3, 6, 5, 4, 3], [3, 9]]>
<tf.RaggedTensor [[0, 1, 2, 3], [2, 3, 4], [3, 6, 5, 4, 3]]>
add a comment |
Turns out it's not flattening it, not exactly sure how it works but it looks like by keeping track of where the row breaks are, and then executes them when evaluated.
tf.RaggedTensorValue(values=array([3, 6, 5, 4, 3, 3, 9, 0, 1, 2, 3]), row_splits=array([ 0, 5, 7, 11]))]
'row_splits' keeps track of where to split the rows.
Here are some results with eager execution.
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
results
<tf.RaggedTensor [[3, 9], [0, 1, 2, 3], [2, 3, 4]]>
<tf.RaggedTensor [[3, 6, 5, 4, 3], [3, 9], [0, 1, 2, 3]]>
<tf.RaggedTensor [[2, 3, 4], [3, 6, 5, 4, 3], [3, 9]]>
<tf.RaggedTensor [[0, 1, 2, 3], [2, 3, 4], [3, 6, 5, 4, 3]]>
Turns out it's not flattening it, not exactly sure how it works but it looks like by keeping track of where the row breaks are, and then executes them when evaluated.
tf.RaggedTensorValue(values=array([3, 6, 5, 4, 3, 3, 9, 0, 1, 2, 3]), row_splits=array([ 0, 5, 7, 11]))]
'row_splits' keeps track of where to split the rows.
Here are some results with eager execution.
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
i, data = iterator.get_next()
#Remove padding
data2= tf.RaggedTensor.from_tensor(data, padding=-1)
print(data2)
results
<tf.RaggedTensor [[3, 9], [0, 1, 2, 3], [2, 3, 4]]>
<tf.RaggedTensor [[3, 6, 5, 4, 3], [3, 9], [0, 1, 2, 3]]>
<tf.RaggedTensor [[2, 3, 4], [3, 6, 5, 4, 3], [3, 9]]>
<tf.RaggedTensor [[0, 1, 2, 3], [2, 3, 4], [3, 6, 5, 4, 3]]>
answered Dec 29 '18 at 23:01
SantoshGupta7SantoshGupta7
6431515
6431515
add a comment |
add a comment |
Thanks for contributing an answer to Stack Overflow!
- Please be sure to answer the question. Provide details and share your research!
But avoid …
- Asking for help, clarification, or responding to other answers.
- Making statements based on opinion; back them up with references or personal experience.
To learn more, see our tips on writing great answers.
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
StackExchange.ready(
function () {
StackExchange.openid.initPostLogin('.new-post-login', 'https%3a%2f%2fstackoverflow.com%2fquestions%2f53973709%2ftensorflow-raggedtensor-from-tensor-flattening-values-from-all-arrays-into-one%23new-answer', 'question_page');
}
);
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Sign up or log in
StackExchange.ready(function () {
StackExchange.helpers.onClickDraftSave('#login-link');
});
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Sign up using Google
Sign up using Facebook
Sign up using Email and Password
Post as a guest
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
Required, but never shown
sFFyHe,MEmSy8dR4InNaeka8RXNO4rhzrEahKA7FBcbddrVI08fP1UO Wd 7cHK maOO,dgSw0zebiQ,j9,Ta