标签：layers Dense keras self 学习计算深度 tf net

层和块

import tensorflow as tf
import numpy as np

net = tf.keras.models.Sequential([
    tf.keras.layers.Dense(256, activation=tf.nn.relu),
    tf.keras.layers.Dense(10)
])

X = tf.random.uniform((2, 20))

net(X)

"""
<tf.Tensor: shape=(2, 10), dtype=float32, numpy=
array([[-0.04290408,  0.16185862,  0.33581334, -0.4772908 , -0.43052262,
        -0.2755071 , -0.01364495, -0.01924989, -0.25683147,  0.14087945],
       [ 0.08288381,  0.25238007,  0.2683103 , -0.31998044, -0.1930172 ,
        -0.47576165, -0.0759996 ,  0.15379383, -0.42053652,  0.01198682]],
      dtype=float32)>
"""

自定义块

class MLP(tf.keras.Model):
    # 用模型参数声明层。这里，我们声明两个全连接的层
    def __init__(self):
        # 调用MLP的父类Model的构造函数来执行必要的初始化
        # 这样，在类实例化时也可以指定其他函数参数，例如模型参数params
        super().__init__()
        # Hidden layer
        self.hidden = tf.keras.layers.Dense(units=256, activation=tf.nn.relu)
        self.out = tf.keras.layers.Dense(units=10)  # Output layer
        
    # 定义模型的前向传播，即如何根据输入X返回所需的模型输出
    def call(self, X):
        return self.out(self.hidden(X))
    
net = MLP()
net(X)

"""
<tf.Tensor: shape=(2, 10), dtype=float32, numpy=
array([[-0.26962578, -0.11410601,  0.07386565,  0.14375813, -0.108279  ,
         0.26483834, -0.03660164, -0.08067364,  0.10585064,  0.24364512],
       [-0.24622872, -0.12058591,  0.06265444, -0.04843516, -0.0802037 ,
        -0.12859441, -0.08167031, -0.00575764,  0.0458674 ,  0.09763081]],
      dtype=float32)>
"""

顺序块

class MySequential(tf.keras.Model):
    def __init__(self, *args):
        super().__init__()
        self.modules = []
        for block in args:
            # 这里block是tf.keras.layers.Layer子类的一个实例
            self.modules.append(block)
            
    def call(self, X):
        for module in self.modules:
            X = module(X)
        return X
    
net = MySequential(
    tf.keras.layers.Dense(units=256, activation=tf.nn.relu),
    tf.keras.layers.Dense(10)
)

net(X)

"""
<tf.Tensor: shape=(2, 10), dtype=float32, numpy=
array([[-1.17208004e-01, -7.67507404e-03,  4.20906067e-01,
         2.63195217e-01,  6.44621402e-02, -1.11673675e-01,
         8.89514387e-02, -1.46050304e-01, -6.07277453e-02,
        -3.42084467e-02],
       [ 2.96452016e-01,  1.08508758e-01,  3.60923648e-01,
         3.58375251e-01,  1.96460947e-01, -9.69487429e-02,
        -3.03983688e-04, -4.94183600e-01,  5.83917871e-02,
         8.36815685e-02]], dtype=float32)>
"""

在前向传播函数中执行代码

class FixedHiddenMLP(tf.keras.Model):
    def __init__(self):
        super().__init__()
        self.flatten = tf.keras.layers.Flatten()
        # 使用tf.constant函数创建的随机权重参数在训练期间不会更新（即为常量参数）
        self.rand_weight = tf.constant(tf.random.uniform((20, 20)))
        self.dense = tf.keras.layers.Dense(20, activation=tf.nn.relu)
        
    def call(self, inputs):
        X = self.flatten(inputs)
        # 使用创建的常量参数以及relu和matmul函数
        """
        我们实现了一个隐藏层， 其权重（self.rand_weight）在实例化时被随机初始化，之后为常量。 
        这个权重不是一个模型参数，因此它永远不会被反向传播更新。 然后，神经网络将这个固定层的输出通过一个全连接层。
        """
        X = tf.nn.relu(tf.matmul(X, self.rand_weight) + 1)
        # 复用全连接层。这相当于两个全连接层共享参数
        X = self.dense(X)
        # 控制流
        while tf.reduce_sum(tf.math.abs(X)) > 1:
            X /= 2
        return tf.reduce_sum(X)
    
net = FixedHiddenMLP()
net(X)

"""
<tf.Tensor: shape=(), dtype=float32, numpy=0.58207244>
"""

class NestMLP(tf.keras.Model):
    def __init__(self):
        super().__init__()
        self.net = tf.keras.Sequential()
        self.net.add(tf.keras.layers.Dense(64, activation=tf.nn.relu))
        self.net.add(tf.keras.layers.Dense(32, activation=tf.nn.relu))
        self.dense = tf.keras.layers.Dense(16, activation=tf.nn.relu)
        
    def call(self, inputs):
        return self.dense(self.net(inputs))
    
chimera = tf.keras.Sequential()
chimera.add(NestMLP())
chimera.add(tf.keras.layers.Dense(20))
chimera.add(FixedHiddenMLP())
chimera(X)

"""
<tf.Tensor: shape=(), dtype=float32, numpy=0.8986759>
"""

参数访问

net = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(4, activation=tf.nn.relu),
    tf.keras.layers.Dense(1)
])

X = tf.random.uniform((2, 4))
net(X)

"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[0.1116072 ],
       [0.02088086]], dtype=float32)>
"""

print(net.layers[2].weights)

"""
[<tf.Variable 'dense_14/kernel:0' shape=(4, 1) dtype=float32, numpy=
array([[ 0.03134096],
       [ 0.51048875],
       [ 0.20824826],
       [-0.5918182 ]], dtype=float32)>, <tf.Variable 'dense_14/bias:0' shape=(1,) dtype=float32, numpy=array([0.], dtype=float32)>]
"""

print(type(net.layers[2].weights[1]))
print(net.layers[2].weights[1])
print(tf.convert_to_tensor(net.layers[2].weights[1]))

"""
<class 'tensorflow.python.ops.resource_variable_ops.ResourceVariable'>
<tf.Variable 'dense_14/bias:0' shape=(1,) dtype=float32, numpy=array([0.], dtype=float32)>
tf.Tensor([0.], shape=(1,), dtype=float32)
"""

print(net.layers[1].weights)
print(net.get_weights())

"""
[<tf.Variable 'dense_13/kernel:0' shape=(4, 4) dtype=float32, numpy=
array([[-0.3207549 ,  0.08903289,  0.43716985,  0.1183672 ],
       [-0.35017866, -0.21974826, -0.167526  , -0.52557135],
       [ 0.6917662 ,  0.45998472,  0.09678668,  0.5109622 ],
       [ 0.7497603 , -0.71563303,  0.4555226 , -0.4751711 ]],
      dtype=float32)>, <tf.Variable 'dense_13/bias:0' shape=(4,) dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>]
[array([[-0.3207549 ,  0.08903289,  0.43716985,  0.1183672 ],
       [-0.35017866, -0.21974826, -0.167526  , -0.52557135],
       [ 0.6917662 ,  0.45998472,  0.09678668,  0.5109622 ],
       [ 0.7497603 , -0.71563303,  0.4555226 , -0.4751711 ]],
      dtype=float32), array([0., 0., 0., 0.], dtype=float32), array([[ 0.03134096],
       [ 0.51048875],
       [ 0.20824826],
       [-0.5918182 ]], dtype=float32), array([0.], dtype=float32)]
"""

net.get_weights()[1]  # 偏置项参数

"""
array([0., 0., 0., 0.], dtype=float32)
"""

def block1(name):
    return tf.keras.Sequential([
        # X是二维的，也就是说没有batch的维度，因此其实没有起作用，经过flatten维度不变
        tf.keras.layers.Flatten(),
        tf.keras.layers.Dense(4, activation=tf.nn.relu, name=name)
    ])
    
def block2():
    net = tf.keras.Sequential()
    for i in range(4):
        # 在这里嵌套
        net.add(block1(name=f'block-{i}'))
    return net

rgnet = tf.keras.Sequential()
rgnet.add(block2())
rgnet.add(tf.keras.layers.Dense(1))
rgnet(X)

"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[0.],
       [0.]], dtype=float32)>
"""

print(rgnet.summary())  # 4*(4+1) + 4*(4+1) + 4*(4+1) + 4*(4+1) + (4+1)

"""
Model: "sequential_16"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
sequential_17 (Sequential)   (2, 4)                    80        
_________________________________________________________________
dense_17 (Dense)             (2, 1)                    5         
=================================================================
Total params: 85
Trainable params: 85
Non-trainable params: 0
_________________________________________________________________
None
"""

rgnet.layers[0].layers[1].layers[1].weights[1]

"""
<tf.Variable 'block-1/bias:0' shape=(4,) dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>
"""

参数初始化

#  内置初始化
net = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(
        4, 
        activation=tf.nn.relu, 
        kernel_initializer=tf.random_normal_initializer(mean=0, stddev=0.01),
        bias_initializer=tf.zeros_initializer()
    ),
    tf.keras.layers.Dense(1)
])

net(X)

"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[ 0.0004508 ],
       [-0.00056325]], dtype=float32)>
"""

net.weights[0], net.weights[1]

"""
(<tf.Variable 'dense_18/kernel:0' shape=(4, 4) dtype=float32, numpy=
 array([[ 0.00936705, -0.00546127,  0.00297332,  0.00961739],
        [ 0.00299437, -0.01580729, -0.00122335,  0.00139136],
        [-0.00579328,  0.01332515,  0.01399793,  0.00444172],
        [-0.00757714, -0.01153165, -0.00556281, -0.00651804]],
       dtype=float32)>,
 <tf.Variable 'dense_18/bias:0' shape=(4,) dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>)
"""

net = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(
        4,
        activation=tf.nn.relu,
        kernel_initializer=tf.keras.initializers.Constant(1),
        bias_initializer=tf.zeros_initializer()
    ),
    tf.keras.layers.Dense(1)
])

net(X)

"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[1.6196353],
       [0.4654615]], dtype=float32)>
"""

net.weights[0], net.weights[1]

"""
(<tf.Variable 'dense_20/kernel:0' shape=(4, 4) dtype=float32, numpy=
 array([[1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.],
        [1., 1., 1., 1.]], dtype=float32)>,
 <tf.Variable 'dense_20/bias:0' shape=(4,) dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>)
"""

net = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(4, activation=tf.nn.relu, kernel_initializer=tf.keras.initializers.GlorotUniform()),
    tf.keras.layers.Dense(1, kernel_initializer=tf.keras.initializers.Constant(1))
])
net(X)

"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[1.1036941 ],
       [0.24121845]], dtype=float32)>
"""

print(net.layers[1].weights[0])
print(net.layers[2].weights[0])

"""
<tf.Variable 'dense_22/kernel:0' shape=(4, 4) dtype=float32, numpy=
array([[ 0.3928185 ,  0.19452113, -0.6222625 ,  0.5098383 ],
       [ 0.7411222 , -0.6079396 ,  0.7660083 ,  0.3188972 ],
       [-0.20573944, -0.5214578 , -0.59844893, -0.67086643],
       [-0.29475045,  0.20483035, -0.06896585,  0.02373922]],
      dtype=float32)>
<tf.Variable 'dense_23/kernel:0' shape=(4, 1) dtype=float32, numpy=
array([[1.],
       [1.],
       [1.],
       [1.]], dtype=float32)>
"""

# 自定义初始化
"""
在这里，我们定义了一个Initializer的子类， 并实现了__call__函数。 该函数返回给定形状和数据类型的所需张量。
"""
class MyInit(tf.keras.initializers.Initializer):
    def __call__(self, shape, dtype=None):
        data = tf.random.uniform(shape, -10, 10, dtype=dtype)
        factor = tf.abs(data) >= 5
        factor = tf.cast(factor, tf.float32)
        return data * factor
    
net = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(4, activation=tf.nn.relu, kernel_initializer=MyInit()),
    tf.keras.layers.Dense(1)
])
net(X)

"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[2.4390702],
       [1.3394781]], dtype=float32)>
"""

print(net.layers[1].weights[0])

"""
<tf.Variable 'dense_24/kernel:0' shape=(4, 4) dtype=float32, numpy=
array([[-0.       ,  9.192398 ,  8.276976 ,  0.       ],
       [-9.562297 ,  0.       ,  6.1546516,  0.       ],
       [ 8.504074 ,  0.       ,  0.       , -7.715745 ],
       [-8.956351 ,  0.       ,  7.2680073,  0.       ]], dtype=float32)>
"""

# 注意，我们始终可以直接设置参数
net.layers[1].weights[0][:].assign(net.layers[1].weights[0] + 1)
net.layers[1].weights[0][0, 0].assign(42)
net.layers[1].weights[0]

"""
<tf.Variable 'dense_24/kernel:0' shape=(4, 4) dtype=float32, numpy=
array([[42.       , 12.192398 , 11.276976 ,  3.       ],
       [-6.562297 ,  3.       ,  9.154652 ,  3.       ],
       [11.504074 ,  3.       ,  3.       , -4.715745 ],
       [-5.9563513,  3.       , 10.268007 ,  3.       ]], dtype=float32)>
"""

参数绑定

"""
有时我们希望在多个层间共享参数： 我们可以定义一个稠密层，然后使用它的参数来设置另一个层的参数。
"""
# tf.keras的表现有点不同，它会自动删除重复层
shared = tf.keras.layers.Dense(4, activation=tf.nn.relu)
net = tf.keras.models.Sequential([
    tf.keras.layers.Flatten(),
    shared,
    shared,
    tf.keras.layers.Dense(1)
])
net(X)

"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[-0.02321801],
       [ 0.01264826]], dtype=float32)>
"""

# 检查参数是否不同
print(len(net.layers) == 3)  # 只剩下flatten，shared，dense三个层

"""
True
"""

延后初始化

net = tf.keras.models.Sequential([
    tf.keras.layers.Dense(256, activation=tf.nn.relu),
    tf.keras.layers.Dense(10)
])

"""
因为输入维数是未知的，所以网络不可能知道输入层权重的维数。 因此，框架尚未初始化任何参数，我们通过尝试访问以下参数进行确认
"""
[net.layers[i].get_weights() for i in range(len(net.layers))]

"""
[[], []]
"""

X = tf.random.uniform((2, 20))
net(X)

"""
<tf.Tensor: shape=(2, 10), dtype=float32, numpy=
array([[-0.11386063, -0.00487196,  0.04176567,  0.1878469 ,  0.10872965,
        -0.16000062, -0.05067926, -0.07579428,  0.08672158, -0.10469792],
       [-0.15152982, -0.07497732,  0.15857704,  0.13723594,  0.32058844,
        -0.15092999,  0.09441049, -0.10225398,  0.17057276, -0.02920698]],
      dtype=float32)>
"""

[w.shape for w in net.get_weights()]

"""
[(20, 256), (256,), (256, 10), (10,)]
"""

不带参数的层

class CenteredLayer(tf.keras.Model):
    def __init__(self):
        super().__init__()
        
    def call(self, inputs):
        return inputs - tf.reduce_mean(inputs)
    
layer = CenteredLayer()
layer(tf.constant([1, 2, 3, 4, 5]))

"""
<tf.Tensor: shape=(5,), dtype=int32, numpy=array([-2, -1,  0,  1,  2])>
"""

net = tf.keras.Sequential([tf.keras.layers.Dense(256), CenteredLayer()])

Y = net(tf.random.uniform((4, 8)))
tf.reduce_mean(Y)

"""
<tf.Tensor: shape=(), dtype=float32, numpy=0.0>
"""

带参数的层

class MyDense(tf.keras.Model):
    def __init__(self, units):
        super().__init__()
        self.units = units
        
    def build(self, X_shape):
        self.weight = self.add_weight(name='weight', shape=[X_shape[-1], self.units], initializer=tf.random_normal_initializer())
        self.bias = self.add_weight(name='bias', shape=[self.units], initializer=tf.zeros_initializer())
        
    def call(self, X):
        linear = tf.matmul(X, self.weight) + self.bias
        return tf.nn.relu(linear)
    
dense = MyDense(3)
dense(tf.random.uniform((2, 5)))
dense.get_weights()

"""
[array([[ 0.00817673,  0.02359941, -0.02445445],
        [-0.01430137,  0.07688206,  0.00117064],
        [-0.01817204, -0.00838405,  0.05307764],
        [ 0.05277959, -0.00456494,  0.01542496],
        [-0.01230057,  0.05387021, -0.00627117]], dtype=float32),
 array([0., 0., 0.], dtype=float32)]
"""

dense(tf.random.uniform((2, 5)))

"""
<tf.Tensor: shape=(2, 3), dtype=float32, numpy=
array([[0.03652133, 0.06277972, 0.00342073],
       [0.00039518, 0.0822235 , 0.02796303]], dtype=float32)>
"""

net = tf.keras.models.Sequential([MyDense(8), MyDense(1)])
net(tf.random.uniform((2, 64)))

"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[0.01318499],
       [0.        ]], dtype=float32)>
"""

加载和保存张量

x = tf.range(4)
np.save('x-file.npy', x)

x2 = np.load('x-file.npy', allow_pickle=True)
x2

"""
array([0, 1, 2, 3])
"""

y = tf.zeros(4)
np.save('xy-files.npy', [x, y])
x2, y2 = np.load('xy-files.npy', allow_pickle=True)
(x2, y2)

"""
(array([0., 1., 2., 3.]), array([0., 0., 0., 0.]))
"""

mydict = {'x': x, 'y': y}
np.save('mydict.npy', mydict)
mydict2 = np.load('mydict.npy', allow_pickle=True)
mydict2

"""
array({'x': <tf.Tensor: shape=(4,), dtype=int32, numpy=array([0, 1, 2, 3])>, 'y': <tf.Tensor: shape=(4,), dtype=float32, numpy=array([0., 0., 0., 0.], dtype=float32)>},
      dtype=object)
"""

class MLP(tf.keras.Model):
    def __init__(self):
        super().__init__()
        self.flatten = tf.keras.layers.Flatten()
        self.hidden = tf.keras.layers.Dense(units=256, activation=tf.nn.relu)
        self.out = tf.keras.layers.Dense(units=10)
        
    def call(self, inputs):
        x = self.flatten(inputs)
        x = self.hidden(x)
        return self.out(x)
    
net = MLP()
X = tf.random.uniform((2, 20))
Y = net(X)
net.save_weights('mlp.params')
clone = MLP()
clone.load_weights('mlp.params')

"""
<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x2688525d7c0>
"""

Y_clone = clone(X)
Y_clone == Y

"""
<tf.Tensor: shape=(2, 10), dtype=bool, numpy=
array([[ True,  True,  True,  True,  True,  True,  True,  True,  True,True], [ True,  True,  True,  True,  True,  True,  True,  True,  True, True]])>
"""

GPU

!nvidia-smi

"""
Sat Apr 22 14:38:27 2023       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 526.47       Driver Version: 526.47       CUDA Version: 12.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name            TCC/WDDM | Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  NVIDIA GeForce ... WDDM  | 00000000:01:00.0  On |                  N/A |
|  0%   47C    P8    30W / 350W |  23905MiB / 24576MiB |      1%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
                                                                               
+-----------------------------------------------------------------------------+
| Processes:                                                                  |
|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
|        ID   ID                                                   Usage      |
|=============================================================================|
|    0   N/A  N/A      3240    C+G   ...me\Application\chrome.exe    N/A      |
|    0   N/A  N/A      7176    C+G   C:\Windows\explorer.exe         N/A      |
|    0   N/A  N/A      7532    C+G   ...7.3-Beta-x64\Snipaste.exe    N/A      |
|    0   N/A  N/A      8276    C+G   ...n1h2txyewy\SearchHost.exe    N/A      |
|    0   N/A  N/A      8316    C+G   ...cw5n1h2txyewy\LockApp.exe    N/A      |
|    0   N/A  N/A      9148    C+G   ...artMenuExperienceHost.exe    N/A      |
|    0   N/A  N/A     11064    C+G   ...2txyewy\TextInputHost.exe    N/A      |
|    0   N/A  N/A     11788    C+G   ...ge\Application\msedge.exe    N/A      |
|    0   N/A  N/A     12380    C+G   ...perience\NVIDIA Share.exe    N/A      |
|    0   N/A  N/A     12668    C+G   ...perience\NVIDIA Share.exe    N/A      |
|    0   N/A  N/A     13468    C+G   ...lPanel\SystemSettings.exe    N/A      |
|    0   N/A  N/A     13956    C+G   ...wekyb3d8bbwe\Video.UI.exe    N/A      |
|    0   N/A  N/A     17232    C+G   ...722.39\msedgewebview2.exe    N/A      |
|    0   N/A  N/A     17620      C   ...\envs\python38\python.exe    N/A      |
|    0   N/A  N/A     20240    C+G   ...e\PhoneExperienceHost.exe    N/A      |
|    0   N/A  N/A     20356    C+G   ...oft\OneDrive\OneDrive.exe    N/A      |
+-----------------------------------------------------------------------------+
"""

tf.device('/CPU:0'), tf.device('/GPU:0')

"""
(<tensorflow.python.eager.context._EagerDeviceContext at 0x26882b41c00>,
 <tensorflow.python.eager.context._EagerDeviceContext at 0x26885287cc0>)
"""

len(tf.config.experimental.list_physical_devices('GPU'))

"""
1
"""

import tensorflow as tf
gpus = tf.config.experimental.list_physical_devices('GPU')
tf.config.experimental.set_virtual_device_configuration(
    gpus[0],
    [
        tf.config.experimental.VirtualDeviceConfiguration(memory_limit=9182),
        tf.config.experimental.VirtualDeviceConfiguration(memory_limit=9182)
    ]
)

len(tf.config.experimental.list_logical_devices('GPU'))

"""
2
"""

def try_gpu(i=0):
    # 如果存在，则返回gpu(i)，否则返回cpu()
    if len(tf.config.experimental.list_logical_devices('GPU')) >= i+1:
        return tf.device(f'/GPU:{i}')
    return tf.device('/CPU:0')

def try_all_gpus():
    # 返回所有可用的GPU，如果没有GPU，则返回CPU
    num_gpus = len(tf.config.experimental.list_logical_devices('GPU'))
    devices = [tf.device(f'/GPU:{i}') for i in range(num_gpus)]
    return devices if devices else [tf.device('/CPU:0')]

try_gpu(), try_gpu(10), try_all_gpus()

"""
(<tensorflow.python.eager.context._EagerDeviceContext at 0x17e9ca0d2c0>,
 <tensorflow.python.eager.context._EagerDeviceContext at 0x17e86ac7a80>,
 [<tensorflow.python.eager.context._EagerDeviceContext at 0x17e9ca9e9c0>,
  <tensorflow.python.eager.context._EagerDeviceContext at 0x17e9cac5740>])
vice
x = tf.constant([1, 2, 3])
"""

x = tf.constant([1, 2, 3])
x.device

"""
'/job:localhost/replica:0/task:0/device:GPU:0'
"""

"""
需要注意的是，无论何时我们要对多个项进行操作， 它们都必须在同一个设备上。 
例如，如果我们对两个张量求和， 我们需要确保两个张量都位于同一个设备上， 
否则框架将不知道在哪里存储结果，甚至不知道在哪里执行计算。
"""
# 存储在GPU上
with try_gpu():
    X = tf.ones((2, 3))
X

"""
<tf.Tensor: shape=(2, 3), dtype=float32, numpy=
array([[1., 1., 1.],
       [1., 1., 1.]], dtype=float32)>
"""

with try_gpu(1):
    Y = tf.random.uniform((2, 3))
Y

"""
<tf.Tensor: shape=(2, 3), dtype=float32, numpy=
array([[0.09042239, 0.88034606, 0.12038887],
       [0.18857598, 0.9658278 , 0.82404184]], dtype=float32)>
"""

with try_gpu(1):
    Z = X
print(X)
print(Z)

"""
tf.Tensor(
[[1. 1. 1.]
 [1. 1. 1.]], shape=(2, 3), dtype=float32)
tf.Tensor(
[[1. 1. 1.]
 [1. 1. 1.]], shape=(2, 3), dtype=float32)
"""

Y + Z

"""
<tf.Tensor: shape=(2, 3), dtype=float32, numpy=
array([[1.0904224, 1.8803461, 1.1203889],
       [1.188576 , 1.9658278, 1.8240418]], dtype=float32)>
"""

"""
假设变量Z已经存在于第二个GPU上。 如果我们仍然在同一个设备作用域
下调用Z2 = Z会发生什么？ 它将返回Z，而不会复制并分配新内存
"""
with try_gpu(1):
    Z2 = Z
Z2 is Z

"""
True
"""

# 类似地，神经网络模型可以指定设备。 下面的代码将模型参数放在GPU上。
strategy = tf.distribute.MirroredStrategy()
with strategy.scope():
    net = tf.keras.models.Sequential([tf.keras.layers.Dense(1)])
    
"""
WARNING:tensorflow:NCCL is not supported when using virtual GPUs, fallingback to reduction to one device
INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:GPU:0', '/job:localhost/replica:0/task:0/device:GPU:1')
"""

net(X)

"""
<tf.Tensor: shape=(2, 1), dtype=float32, numpy=
array([[-1.265375],
       [-1.265375]], dtype=float32)>
"""

net.layers[0].weights[0].device, net.layers[0].weights[1].device

"""
('/job:localhost/replica:0/task:0/device:GPU:0',
 '/job:localhost/replica:0/task:0/device:GPU:0')
"""

标签：layers,Dense,keras,self,学习,计算,深度,tf,net
From： https://www.cnblogs.com/lotuslaw/p/17343160.html

5.深度学习计算

层和块

自定义块

顺序块

在前向传播函数中执行代码

参数访问

参数初始化

参数绑定

延后初始化

不带参数的层

带参数的层

加载和保存张量

GPU

相关文章

赞助商

阅读排行