📄 transformer.cpython-312.pyc
/home/palash/git/iron_learn/python_scripts/transformer/__pycache__/transformer.cpython-312.pyc
Language: pyc • Lines: 514
�

�L�i�9��r�ddlZddlmZmZmZ	ejjd�j�e	d�Gd�d�ZGd�d�Z
Gd	�d
�ZGd�d�ZGd
�d�ZGd�d�ZGd�d�ZGd�d�ZGd�d�Zy#e
$rZe	de���YdZ[�pdZ[wwxYw)�N)�Linear�ReLU�Softmaxz(GPU found and will be used for training.zGPU not found or error: c��eZdZd�Zd�Zd�Zy)�	Embeddingc��||_||_tjj	||�tj
d|z�z|_y)N�@)�
vocab_size�d_model�np�random�randn�sqrt�weight)�selfr
rs   �E/home/palash/git/iron_learn/python_scripts/transformer/transformer.py�__init__zEmbedding.__init__s9��$�������i�i�o�o�j�'�:�R�W�W�S�:�EU�=V�V���c�Z�|jdk(sJd��||_|j|S)N��+Input should be a 1D array of token indices)�ndim�inputsr�r�xs  r�forwardzEmbedding.forwards-���v�v��{�I�I�I�{�����{�{�1�~�rc�v�tjj|j|j||z�y�N)r�add�atrr�r�grad_output�
learning_rates   r�backwardzEmbedding.backwards(��
���	�	�$�+�+�t�{�{�]�N�[�,H�I�rN��__name__�
__module__�__qualname__rrr$�rrrr
s��W�
�
rrc��eZdZd�Zd�Zd�Zy)�PositionalEncodingc��||_||_tjj	|j|j�dz|_y)Ng{�G�z�?)�embedding_dimension�sequence_lengthrr
rr)rr-r.s   rrzPositionalEncoding.__init__s:��#6�� �.����i�i�o�o�d�&:�&:�D�<T�<T�U�X\�\��rc��|jdk(r|jd|jk(sJd��||_||jd|jd�dd�fzS)N�rz?Input should be of shape (sequence_length, embedding_dimension)r)r�shaper-rrrs  rrzPositionalEncoding.forwardsc���v�v��{�q�w�w�q�z�T�-E�-E�E�	I�HI�	I�E�����4�;�;�{�����
�{�A�~�.�.�.rc�Z�|jd}|jd|xxx||zzccc|S�Nr)r1r)rr"r#�seq_lens    rr$zPositionalEncoding.backward$s3���#�#�A�&�����H�W����!<�<���rNr%r)rrr+r+s��]�
/�
rr+c�"�eZdZdd�Zdd�Zd�Zy)�Dropoutc� �||_d|_yr)�p�mask)rr8s  rrzDropout.__init__*s�������	rc���|r|jdk(r|Stjj|j�|jkD|_||j
zd|jz
zS)Nr��?)r8rr
�randr1r9)rr�trainings   rrzDropout.forward.sR���4�6�6�Q�;��H��Y�Y�^�^�Q�W�W�-����6��	��D�I�I�
�#����,�/�/rc�\�|j�|S||jzd|jz
zS)Nr;)r9r8�rr"s  rr$zDropout.backward7s.���9�9�����d�i�i�'�C�$�&�&�L�9�9rN�g�������?�Tr%r)rrr6r6)s���0�:rr6c��eZdZd�Zd�Zd�Zy)�	Attentionc��||zdk(sJd��||_||_||z|_t||�|_t||�|_t||�|_t||�|_t|�D�cgc]}t���c}|_
d|_d|_d|_
i|_d|_d|_ycc}w)Nrz&d_model must be divisible by num_heads)r�	num_heads�d_kr�W_q�W_k�W_v�W_o�ranger�softmax_layers�q_full�k_full�v_full�attention_probsr�causal_mask)rrrE�_s    rrzAttention.__init__=s�����"�a�'�Q�)Q�Q�'����"����i�'����'�7�+����'�7�+����'�7�+����'�7�+���27�	�2B�C�Q�w�y�C������������!����������Ds�9B=c��|jdk(r|jd|jk(sJd��||_|jj|�|_|jj|�|_|jj|�|_
tjtj|jd|jdf�dzd��|_g}t|j �D�]}||j"z|dz|j"z}}|jdd�||�f}|jdd�||�f}|jdd�||�f}tj$||j&�tj(|j"�z}	|	|jz
}	|j*|j|	�|j,|<tj$|j,||�}
|j/|
���tj0|d��}|j2j|�S)	Nr0rz3Input should be of shape (sequence_length, d_model)r�e�����k�������axis)rr1rrrGrrMrHrNrIrOr�triu�onesrQrKrErF�dot�TrrLrP�append�concatenaterJ)rr�head_outputs�i�start�end�qrV�v�attention_scores�head_output�attention_outputs            rrzAttention.forwardQs����v�v��{�q�w�w�q�z�T�\�\�9�p�;p�p�9�����h�h�&�&�q�)����h�h�&�&�q�)����h�h�&�&�q�)����7�7�2�7�7�A�G�G�A�J�����
�+C�#D�t�#K�q�Q������t�~�~�&�	-�A��T�X�X���A�����'9�3�E����A�u�S�y�L�)�A����A�u�S�y�L�)�A����A�u�S�y�L�)�A�!�v�v�a����~�������0A�A���� 0� 0�0��&*�&9�&9�!�&<�&D�&D�EU�&V�D� � ��#��&�&��!5�!5�a�!8�!�<�K�����,�	-��>�>�,�R�@���x�x��� 0�1�1rc��|jj||�}tj|j�}tj|j
�}tj|j�}t|j�D�]^}||jz|dz|jz}	}|dd�||	�f}
|jdd�||	�f|j
dd�||	�f|jdd�||	�f}
}}|j|}tj|j|
�|dd�||	�f<tj|
|
j�}|j|j|�tj|j�z}||jdk(z}tj||�|dd�||	�f<tj|j|�|dd�||	�f<��a|j j||�}|j"j||�}|j$j||�}||z|zS)Nrr)rJr$r�
zeros_likerMrNrOrKrErFrPr\r]rLrrQrGrHrI)rr"r#�grad_attention_output�grad_q_full�grad_k_full�grad_v_fullrarbrc�grad_head_outputrdrVre�probs�
grad_probs�grad_scores�grad_input_q�grad_input_k�grad_input_vs                    rr$zAttention.backwardns�� $��� 1� 1�+�}� M���m�m�D�K�K�0���m�m�D�K�K�0���m�m�D�K�K�0���t�~�~�&�	A�A��T�X�X���A�����'9�3�E�4�Q��c�	�\�B���k�k�!�U�3�Y�,�/����Q��c�	�\�1J�D�K�K�XY�[`�ad�[d�Xd�Le�!�q�A��(�(��+�E�)+���u�w�w�8H�(I�K��5��9��%���� 0�!�#�#�6�J��-�-�a�0�9�9�*�E����PT�PX�PX�HY�Y�K�%��)9�)9�Q�)>�?�K�)+���{�A�(>�K��5��9��%�(*���{�}�}�a�(@�K��5��9��%�'	A�*�x�x�(�(��m�D���x�x�(�(��m�D���x�x�(�(��m�D���l�*�\�9�9rNr%r)rrrCrC<s�� �(2�: :rrCc��eZdZd�Zd�Zd�Zy)�Attention_NDc���||zdk(sJ�||_||_||z|_t||�|_t||�|_t||�|_t||�|_t�|_	yr3)
rrErFrrGrHrIrJr�softmax)rrrEs   rrzAttention_ND.__init__�sp����"�a�'�'�'����"����i�'����'�7�+����'�7�+����'�7�+����'�7�+����y��rc��||_|jd}|jj|�}|jj|�}|j
j|�}|j
||j|j�jddd�|_
|j
||j|j�jddd�|_|j
||j|j�jddd�|_tj|j|jjddd��tj|j�z}tj tj"||f�dzd��}||z
}|j$j|�|_tj|j&|j�}|jddd�j
||j(�}|j*j|�S)Nrrr0rTrU)rr1rGrrHrI�reshaperErF�	transposerdrVrer�matmulrrZr[ryrprrJ)	rrr4rdrVre�scoresr9�outs	         rrzAttention_ND.forward�s�������'�'�!�*��
�H�H���Q����H�H���Q����H�H���Q������7�D�N�N�D�H�H�=�G�G��1�a�P������7�D�N�N�D�H�H�=�G�G��1�a�P������7�D�N�N�D�H�H�=�G�G��1�a�P������4�6�6�4�6�6�#3�#3�A�q�!�#<�=�������@Q�Q���w�w�r�w�w���1�2�T�9�Q�?���$����\�\�)�)�&�1��
��i�i��
�
�D�F�F�+���m�m�A�q�!�$�,�,�W�d�l�l�C���x�x����$�$rc�^�|jj||�}|jd}|j||j|j
�j
ddd�}tj|jj
ddd�|�}tj||jj
ddd��}|jj|�tj|j
�z}tj||j�}tj|j
ddd�|j�}	|j
ddd�j||j�}|	j
ddd�j||j�}	|j
ddd�j||j�}|j j||�|j"j|	|�z|j$j||�zS)Nrrr0)rJr$r1r{rErFr|rr}rpreryrrVrdrrGrHrI)
rr"r#�grad_outr4�grad_vrqrr�grad_q�grad_ks
          rr$zAttention_ND.backward�s����8�8�$�$�[�-�@���.�.��#���#�#�G�T�^�^�T�X�X�F�P�P�QR�TU�WX�Y�����4�:�:�/�/��1�a�8�(�C���Y�Y�x����)9�)9�!�Q��)B�C�
��l�l�+�+�J�7�"�'�'�$�(�(�:K�K�����;����/�����;�0�0��A�q�9�4�6�6�B���!�!�!�Q��*�2�2�7�D�L�L�I���!�!�!�Q��*�2�2�7�D�L�L�I���!�!�!�Q��*�2�2�7�D�L�L�I�����!�!�&�-�8����!�!�&�-�8�9����!�!�&�-�8�9�	:rNr%r)rrrwrw�s��
!�%�@:rrwc��eZdZd�Zd�Zd�Zy)�FeedForwardc�f�t||�|_t�|_t||�|_yr)r�linear1r�relu�linear2)rr�d_ffs   rrzFeedForward.__init__�s'���g�t�,����F��	��d�G�,��rc��|jj|�}|jj|�}|jj|�}|Sr)r�rr�r�rs  rrzFeedForward.forward�sC���L�L� � ��#���I�I���a� ���L�L� � ��#���rc��|jj||�}|jj|�}|jj||�}|Sr)r�r$r�r�r!s   rr$zFeedForward.backward�sH���l�l�+�+�K��G���i�i�(�(��5���l�l�+�+�K��G���rNr%r)rrr�r��s��-��rr�c��eZdZd�Zd�Zd�Zy)�ResidualConnectionc��yrr))rs rrzResidualConnection.__init__�s��rc��||zSrr))rr�sublayer_outputs   rrzResidualConnection.forward�s
���?�"�"rc�
�||fSrr)r?s  rr$zResidualConnection.backward�s
���K�'�'rNr%r)rrr�r��s��
�#�(rr�c� �eZdZdd�Zd�Zd�Zy)�	LayerNormc��||_||_tj|�|_tj
|�|_yr)r�epsrr[�gamma�zeros�beta)rrr�s   rrzLayerNorm.__init__�s0���������W�W�W�%��
��H�H�W�%��	rc�R�||_tj|dd��|_tj|dd��|_||jz
tj|j|j
z�z|_|j|jz|jzS)NrWT�rY�keepdims)	rr�mean�varrr��x_normalizedr�r�rs  rrzLayerNorm.forwardsz������G�G�A�B��6��	��6�6�!�"�t�4�������]�b�g�g�d�h�h����6I�.J�J����z�z�D�-�-�-��	�	�9�9rc�x�tj||jzd��}tj|d��}||jz}|j}tj||j
|jz
zdz|j|jzdzzdd��}|d|zztj|j
|jz
dd��z}tj|d	ztj|j|jz�zdd��}	|	|z}
|tj|j|jz�z}||d
z|j
|jz
z|zz
}||
|zz
}|xj||zzc_|xj||zzc_
|S)NrrXg�g��rWTr�g�g�r	)r�sumr�r�rrr�r�r�rr�)rr"r#�
grad_gamma�	grad_beta�grad_x_normalized�N�grad_var�grad_mean_from_var�grad_mean_direct�	grad_mean�
grad_inputs            rr$zLayerNorm.backwards����V�V�K�$�*;�*;�;�!�D�
��F�F�;�Q�/�	�(�$�*�*�4��
�L�L���6�6�+�t�{�{�T�Y�Y�/F�G�$�N�RV�RZ�RZ�]a�]e�]e�Re�ko�Qp�p�wy�EI�J��%����2�R�V�V�D�K�K�$�)�)�<S�Z\�gk�5l�l���6�6�"3�d�":�R�W�W�T�X�X�PT�PX�PX�EX�=Y�"Y�`b�mq�r��$�'9�9�	�'�������D�H�H�1D�)E�E�
��h��n����d�i�i�(?�@�1�D�D�
��i�!�m�#�
�	
�
�
�m�j�0�0�
��	�	�]�Y�.�.�	��rN)g�����ư>r%r)rrr�r��s��&�:�rr�c�"�eZdZdd�Zdd�Zd�Zy)�Transformerc��||_||_t||�|_t	||�|_t
|�|_t||�|_	t
|�|_
t
|�|_t||�|_
t�|_t!|�|_t�|_t!|�|_t)||�|_yr)r
rr�	embeddingr+�positional_encodingr6�emb_dropoutrw�	attention�dropout1�dropout2r��feed_forwardr��	residual1r��norm1�	residual2�norm2r�output_projection)rr
rrEr�r.�	dropout_ps       rrzTransformer.__init__"s���$������"�:�w�7���#5�g��#O�� �"�9�-���%�g�y�9����	�*��
��	�*��
�'���6���+�-����w�'��
�+�-����w�'��
�!'���!<��rc���|jdk(sJd��|jj|�}|jj|�}|jj||��}||_|jj|�}|jj||��}|jj||�}|jj|�}||_
|jj|�}|jj||��}|jj||�}|jj|�}||_|j j|�}|S)Nrr)r=)rr�rr�r��pos_encodedr�r�r�r��after_norm1r�r�r�r��after_norm2r�)rrr=�attn_output�	ff_output�logitss      rrzTransformer.forward3sF���v�v��{�I�I�I�{��N�N�"�"�1�%���$�$�,�,�Q�/�����$�$�Q��$�:������n�n�,�,�Q�/���m�m�+�+�K�(�+�K���N�N�"�"�1�k�2���J�J���q�!������%�%�-�-�a�0�	��M�M�)�)�)�h�)�G�	��N�N�"�"�1�i�0���J�J���q�!������'�'�/�/��2���
rc��|jj||�}|jj||�}|jj|�\}}|jj|�}|j
j||�}||z}|jj||�}|jj|�\}	}
|jj|
�}
|jj|
|�}||	z}|jj||�}|jj|�}|jj||�yr)
r�r$r�r�r�r�r�r�r�r�r�r�r�)
rr"r#�grad_after_norm2�grad_residual2_input�grad_x_before_ff�grad_ff_output�grad_after_norm1�grad_residual1_input�grad_pos_encodedrk�grad_attention_input�grad_embedding_outputs
             rr$zTransformer.backwardMsQ���1�1�:�:�;�
�V�� $�z�z�2�2�3C�]�S��,0�>�>�+B�+B�CW�+X�(��.����/�/��?�� �,�,�5�5�n�m�T��,�.>�>�� $�z�z�2�2�3C�]�S��37�.�.�2I�2I�J^�2_�/��/� $�
�
� 6� 6�7L� M�� $�~�~�6�6�7L�m�\�� 4�6F�F��!%� 8� 8� A� A�BV�Xe� f�� $� 0� 0� 9� 9�:O� P��	
����� 5�}�ErNr@rAr%r)rrr�r�!s��=�"�4&Frr�)�cupyr�
simple_layersrrr�cuda�Device�use�print�	Exception�err+r6rCrwr�r�r�r�r)rr�<module>r�s����/�/�*��G�G�N�N�1�����	�
4�5�
�
��� :�:�&R:�R:�hJ:�J:�X��&(�(�&�&�PRF�RF��u�*�	�$�Q�C�
(�)�)��*�s�1B�B6�!B1�1B6