Mr.Combet Webshell 🔥

Edit File Name : token.html

<!DOCTYPE html>

<title>token — Constants used with Python parse trees &#8212; Python 3.12.3 documentation</title><meta name="viewport" content="width=device-width, initial-scale=1.0">
 
 <link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=80d5e7a1" />
 <link rel="stylesheet" type="text/css" href="../_static/pydoctheme.css?v=bb723527" />
 <link id="pygments_dark_css" media="(prefers-color-scheme: dark)" rel="stylesheet" type="text/css" href="../_static/pygments_dark.css?v=b20cc3f5" />
 
 <script src="../_static/documentation_options.js?v=2c828074"></script>
 <script src="../_static/doctools.js?v=888ff710"></script>
 <script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
 
 <script src="../_static/sidebar.js"></script>
 
 <link rel="search" type="application/opensearchdescription+xml"
 title="Search within Python 3.12.3 documentation"
 href="../_static/opensearch.xml"/>
 <link rel="author" title="About these documents" href="../about.html" />
 <link rel="index" title="Index" href="../genindex.html" />
 <link rel="search" title="Search" href="../search.html" />
 <link rel="copyright" title="Copyright" href="../copyright.html" />
 <link rel="next" title="keyword — Testing for Python keywords" href="keyword.html" />
 <link rel="prev" title="symtable — Access to the compiler’s symbol tables" href="symtable.html" />
 <link rel="canonical" href="https://docs.python.org/3/library/token.html" />

</head>
<body>
<div class="mobile-nav">
 <input type="checkbox" id="menuToggler" class="toggler__input" aria-controls="navigation"
 aria-pressed="false" aria-expanded="false" role="button" aria-label="Menu" />
 <nav class="nav-content" role="navigation">
 <label for="menuToggler" class="toggler__label">
 
 </label>
 
 <a href="https://www.python.org/" class="nav-logo">
 <img src="../_static/py.svg" alt="Python logo"/>
 </a>
 
 <form role="search" class="search" action="../search.html" method="get">
 <svg xmlns="http://www.w3.org/2000/svg" width="20" height="20" viewBox="0 0 24 24" class="search-icon">
 <path fill-rule="nonzero" fill="currentColor" d="M15.5 14h-.79l-.28-.27a6.5 6.5 0 001.48-5.34c-.47-2.78-2.79-5-5.59-5.34a6.505 6.505 0 00-7.27 7.27c.34 2.8 2.56 5.12 5.34 5.59a6.5 6.5 0 005.34-1.48l.27.28v.79l4.25 4.25c.41.41 1.08.41 1.49 0 .41-.41.41-1.08 0-1.49L15.5 14zm-6 0C7.01 14 5 11.99 5 9.5S7.01 5 9.5 5 14 7.01 14 9.5 11.99 14 9.5 14z"></path>
 </svg>
 <input placeholder="Quick search" aria-label="Quick search" type="search" name="q" />
 <input type="submit" value="Go"/>
 </form>
 
 </nav>
 <div class="menu-wrapper">
 <nav class="menu" role="navigation" aria-label="main navigation">
 <div class="language_switcher_placeholder"></div>
 
<label class="theme-selector-label">
 Theme
 <select class="theme-selector" oninput="activateTheme(this.value)">
 <option value="auto" selected>Auto</option>
 <option value="light">Light</option>
 <option value="dark">Dark</option>
 </select>
</label>
 <div>
 <h4>Previous topic</h4>
 <a href="symtable.html"
 title="previous chapter"><code class="xref py py-mod docutils literal notranslate">symtable</code> — Access to the compiler’s symbol tables</a>
 </div>
 <div>
 <h4>Next topic</h4>
 <a href="keyword.html"
 title="next chapter"><code class="xref py py-mod docutils literal notranslate">keyword</code> — Testing for Python keywords</a>
 </div>
 <div role="note" aria-label="source link">
 <h3>This Page</h3>
 <ul class="this-page-menu">
 <li><a href="../bugs.html">Report a Bug</a></li>
 <li>
 <a href="https://github.com/python/cpython/blob/main/Doc/library/token.rst"
 rel="nofollow">Show Source
 </a>
 </li>
 </ul>
 </div>
 </nav>
 </div>
</div>

<div class="related" role="navigation" aria-label="related navigation">
 <h3>Navigation</h3>
 <ul>
 <li class="right" style="margin-right: 10px">
 <a href="../genindex.html" title="General Index"
 accesskey="I">index</a></li>
 <li class="right" >
 <a href="../py-modindex.html" title="Python Module Index"
 >modules</a> |</li>
 <li class="right" >
 <a href="keyword.html" title="keyword — Testing for Python keywords"
 accesskey="N">next</a> |</li>
 <li class="right" >
 <a href="symtable.html" title="symtable — Access to the compiler’s symbol tables"
 accesskey="P">previous</a> |</li>

<li><img src="../_static/py.svg" alt="Python logo" style="vertical-align: middle; margin-top: -1px"/></li>
 <li><a href="https://www.python.org/">Python</a> &#187;</li>
 <li class="switchers">
 <div class="language_switcher_placeholder"></div>
 <div class="version_switcher_placeholder"></div>
 </li>
 <li>
 
 </li>
 <li id="cpython-language-and-version">
 <a href="../index.html">3.12.3 Documentation</a> &#187;
 </li>

<li class="nav-item nav-item-1"><a href="index.html" >The Python Standard Library</a> &#187;</li>
 <li class="nav-item nav-item-2"><a href="language.html" accesskey="U">Python Language Services</a> &#187;</li>
 <li class="nav-item nav-item-this"><a href=""><code class="xref py py-mod docutils literal notranslate">token</code> — Constants used with Python parse trees</a></li>
 <li class="right">

<div class="document">
 <div class="documentwrapper">
 <div class="bodywrapper">
 <div class="body" role="main">
 
 <section id="module-token">
<h1><a class="reference internal" href="#module-token" title="token: Constants representing terminal nodes of the parse tree."><code class="xref py py-mod docutils literal notranslate">token</code></a> — Constants used with Python parse trees<a class="headerlink" href="#module-token" title="Link to this heading">¶</a></h1>
Source code: <a class="reference external" href="https://github.com/python/cpython/tree/3.12/Lib/token.py">Lib/token.py</a>
<hr class="docutils" />
This module provides constants which represent the numeric values of leaf nodes
of the parse tree (terminal tokens). Refer to the file <code class="file docutils literal notranslate">Grammar/Tokens</code>
in the Python distribution for the definitions of the names in the context of
the language grammar. The specific numeric values which the names map to may
change between Python versions.
The module also provides a mapping from numeric codes to names and some
functions. The functions mirror definitions in the Python C header files.
<dl class="py data">
<dt class="sig sig-object py" id="token.tok_name">
token.tok_name<a class="headerlink" href="#token.tok_name" title="Link to this definition">¶</a></dt>
<dd>Dictionary mapping the numeric values of the constants defined in this module
back to name strings, allowing more human-readable representation of parse trees
to be generated.
</dd></dl>

<dl class="py function">
<dt class="sig sig-object py" id="token.ISTERMINAL">
token.ISTERMINAL(x)<a class="headerlink" href="#token.ISTERMINAL" title="Link to this definition">¶</a></dt>
<dd>Return <code class="docutils literal notranslate">True</code> for terminal token values.
</dd></dl>

<dl class="py function">
<dt class="sig sig-object py" id="token.ISNONTERMINAL">
token.ISNONTERMINAL(x)<a class="headerlink" href="#token.ISNONTERMINAL" title="Link to this definition">¶</a></dt>
<dd>Return <code class="docutils literal notranslate">True</code> for non-terminal token values.
</dd></dl>

<dl class="py function">
<dt class="sig sig-object py" id="token.ISEOF">
token.ISEOF(x)<a class="headerlink" href="#token.ISEOF" title="Link to this definition">¶</a></dt>
<dd>Return <code class="docutils literal notranslate">True</code> if x is the marker indicating the end of input.
</dd></dl>

The token constants are:
<dl class="py data">
<dt class="sig sig-object py" id="token.ENDMARKER">
token.ENDMARKER<a class="headerlink" href="#token.ENDMARKER" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.NAME">
token.NAME<a class="headerlink" href="#token.NAME" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.NUMBER">
token.NUMBER<a class="headerlink" href="#token.NUMBER" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.STRING">
token.STRING<a class="headerlink" href="#token.STRING" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.NEWLINE">
token.NEWLINE<a class="headerlink" href="#token.NEWLINE" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.INDENT">
token.INDENT<a class="headerlink" href="#token.INDENT" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.DEDENT">
token.DEDENT<a class="headerlink" href="#token.DEDENT" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.LPAR">
token.LPAR<a class="headerlink" href="#token.LPAR" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;(&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.RPAR">
token.RPAR<a class="headerlink" href="#token.RPAR" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;)&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.LSQB">
token.LSQB<a class="headerlink" href="#token.LSQB" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;[&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.RSQB">
token.RSQB<a class="headerlink" href="#token.RSQB" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;]&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.COLON">
token.COLON<a class="headerlink" href="#token.COLON" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;:&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.COMMA">
token.COMMA<a class="headerlink" href="#token.COMMA" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;,&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.SEMI">
token.SEMI<a class="headerlink" href="#token.SEMI" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;;&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.PLUS">
token.PLUS<a class="headerlink" href="#token.PLUS" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;+&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.MINUS">
token.MINUS<a class="headerlink" href="#token.MINUS" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;-&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.STAR">
token.STAR<a class="headerlink" href="#token.STAR" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;*&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.SLASH">
token.SLASH<a class="headerlink" href="#token.SLASH" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;/&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.VBAR">
token.VBAR<a class="headerlink" href="#token.VBAR" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;|&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.AMPER">
token.AMPER<a class="headerlink" href="#token.AMPER" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;&amp;&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.LESS">
token.LESS<a class="headerlink" href="#token.LESS" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;&lt;&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.GREATER">
token.GREATER<a class="headerlink" href="#token.GREATER" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;&gt;&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.EQUAL">
token.EQUAL<a class="headerlink" href="#token.EQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.DOT">
token.DOT<a class="headerlink" href="#token.DOT" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;.&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.PERCENT">
token.PERCENT<a class="headerlink" href="#token.PERCENT" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;%&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.LBRACE">
token.LBRACE<a class="headerlink" href="#token.LBRACE" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;{&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.RBRACE">
token.RBRACE<a class="headerlink" href="#token.RBRACE" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;}&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.EQEQUAL">
token.EQEQUAL<a class="headerlink" href="#token.EQEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;==&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.NOTEQUAL">
token.NOTEQUAL<a class="headerlink" href="#token.NOTEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;!=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.LESSEQUAL">
token.LESSEQUAL<a class="headerlink" href="#token.LESSEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;&lt;=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.GREATEREQUAL">
token.GREATEREQUAL<a class="headerlink" href="#token.GREATEREQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;&gt;=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.TILDE">
token.TILDE<a class="headerlink" href="#token.TILDE" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;~&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.CIRCUMFLEX">
token.CIRCUMFLEX<a class="headerlink" href="#token.CIRCUMFLEX" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;^&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.LEFTSHIFT">
token.LEFTSHIFT<a class="headerlink" href="#token.LEFTSHIFT" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;&lt;&lt;&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.RIGHTSHIFT">
token.RIGHTSHIFT<a class="headerlink" href="#token.RIGHTSHIFT" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;&gt;&gt;&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.DOUBLESTAR">
token.DOUBLESTAR<a class="headerlink" href="#token.DOUBLESTAR" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;**&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.PLUSEQUAL">
token.PLUSEQUAL<a class="headerlink" href="#token.PLUSEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;+=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.MINEQUAL">
token.MINEQUAL<a class="headerlink" href="#token.MINEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;-=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.STAREQUAL">
token.STAREQUAL<a class="headerlink" href="#token.STAREQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;*=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.SLASHEQUAL">
token.SLASHEQUAL<a class="headerlink" href="#token.SLASHEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;/=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.PERCENTEQUAL">
token.PERCENTEQUAL<a class="headerlink" href="#token.PERCENTEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;%=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.AMPEREQUAL">
token.AMPEREQUAL<a class="headerlink" href="#token.AMPEREQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;&amp;=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.VBAREQUAL">
token.VBAREQUAL<a class="headerlink" href="#token.VBAREQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;|=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.CIRCUMFLEXEQUAL">
token.CIRCUMFLEXEQUAL<a class="headerlink" href="#token.CIRCUMFLEXEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;^=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.LEFTSHIFTEQUAL">
token.LEFTSHIFTEQUAL<a class="headerlink" href="#token.LEFTSHIFTEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;&lt;&lt;=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.RIGHTSHIFTEQUAL">
token.RIGHTSHIFTEQUAL<a class="headerlink" href="#token.RIGHTSHIFTEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;&gt;&gt;=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.DOUBLESTAREQUAL">
token.DOUBLESTAREQUAL<a class="headerlink" href="#token.DOUBLESTAREQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;**=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.DOUBLESLASH">
token.DOUBLESLASH<a class="headerlink" href="#token.DOUBLESLASH" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;//&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.DOUBLESLASHEQUAL">
token.DOUBLESLASHEQUAL<a class="headerlink" href="#token.DOUBLESLASHEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;//=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.AT">
token.AT<a class="headerlink" href="#token.AT" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;&#64;&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.ATEQUAL">
token.ATEQUAL<a class="headerlink" href="#token.ATEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;&#64;=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.RARROW">
token.RARROW<a class="headerlink" href="#token.RARROW" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;-&gt;&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.ELLIPSIS">
token.ELLIPSIS<a class="headerlink" href="#token.ELLIPSIS" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;...&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.COLONEQUAL">
token.COLONEQUAL<a class="headerlink" href="#token.COLONEQUAL" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;:=&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.EXCLAMATION">
token.EXCLAMATION<a class="headerlink" href="#token.EXCLAMATION" title="Link to this definition">¶</a></dt>
<dd>Token value for <code class="docutils literal notranslate">&quot;!&quot;</code>.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.OP">
token.OP<a class="headerlink" href="#token.OP" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.AWAIT">
token.AWAIT<a class="headerlink" href="#token.AWAIT" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.ASYNC">
token.ASYNC<a class="headerlink" href="#token.ASYNC" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.TYPE_IGNORE">
token.TYPE_IGNORE<a class="headerlink" href="#token.TYPE_IGNORE" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.TYPE_COMMENT">
token.TYPE_COMMENT<a class="headerlink" href="#token.TYPE_COMMENT" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.SOFT_KEYWORD">
token.SOFT_KEYWORD<a class="headerlink" href="#token.SOFT_KEYWORD" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.FSTRING_START">
token.FSTRING_START<a class="headerlink" href="#token.FSTRING_START" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.FSTRING_MIDDLE">
token.FSTRING_MIDDLE<a class="headerlink" href="#token.FSTRING_MIDDLE" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.FSTRING_END">
token.FSTRING_END<a class="headerlink" href="#token.FSTRING_END" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.COMMENT">
token.COMMENT<a class="headerlink" href="#token.COMMENT" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.NL">
token.NL<a class="headerlink" href="#token.NL" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.ERRORTOKEN">
token.ERRORTOKEN<a class="headerlink" href="#token.ERRORTOKEN" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.N_TOKENS">
token.N_TOKENS<a class="headerlink" href="#token.N_TOKENS" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.NT_OFFSET">
token.NT_OFFSET<a class="headerlink" href="#token.NT_OFFSET" title="Link to this definition">¶</a></dt>
<dd></dd></dl>

The following token type values aren’t used by the C tokenizer but are needed for
the <a class="reference internal" href="tokenize.html#module-tokenize" title="tokenize: Lexical scanner for Python source code."><code class="xref py py-mod docutils literal notranslate">tokenize</code></a> module.
<dl class="py data">
<dt class="sig sig-object py">
token.COMMENT</dt>
<dd>Token value used to indicate a comment.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py">
token.NL</dt>
<dd>Token value used to indicate a non-terminating newline. The
<a class="reference internal" href="#token.NEWLINE" title="token.NEWLINE"><code class="xref py py-data docutils literal notranslate">NEWLINE</code></a> token indicates the end of a logical line of Python code;
<code class="docutils literal notranslate">NL</code> tokens are generated when a logical line of code is continued over
multiple physical lines.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py" id="token.ENCODING">
token.ENCODING<a class="headerlink" href="#token.ENCODING" title="Link to this definition">¶</a></dt>
<dd>Token value that indicates the encoding used to decode the source bytes
into text. The first token returned by <a class="reference internal" href="tokenize.html#tokenize.tokenize" title="tokenize.tokenize"><code class="xref py py-func docutils literal notranslate">tokenize.tokenize()</code></a> will
always be an <code class="docutils literal notranslate">ENCODING</code> token.
</dd></dl>

<dl class="py data">
<dt class="sig sig-object py">
token.TYPE_COMMENT</dt>
<dd>Token value indicating that a type comment was recognized. Such
tokens are only produced when <a class="reference internal" href="ast.html#ast.parse" title="ast.parse"><code class="xref py py-func docutils literal notranslate">ast.parse()</code></a> is invoked with
<code class="docutils literal notranslate">type_comments=True</code>.
</dd></dl>

<div class="versionchanged">
Changed in version 3.5: Added <a class="reference internal" href="#token.AWAIT" title="token.AWAIT"><code class="xref py py-data docutils literal notranslate">AWAIT</code></a> and <a class="reference internal" href="#token.ASYNC" title="token.ASYNC"><code class="xref py py-data docutils literal notranslate">ASYNC</code></a> tokens.
</div>
<div class="versionchanged">
Changed in version 3.7: Added <a class="reference internal" href="#token.COMMENT" title="token.COMMENT"><code class="xref py py-data docutils literal notranslate">COMMENT</code></a>, <a class="reference internal" href="#token.NL" title="token.NL"><code class="xref py py-data docutils literal notranslate">NL</code></a> and <a class="reference internal" href="#token.ENCODING" title="token.ENCODING"><code class="xref py py-data docutils literal notranslate">ENCODING</code></a> tokens.
</div>
<div class="versionchanged">
Changed in version 3.7: Removed <a class="reference internal" href="#token.AWAIT" title="token.AWAIT"><code class="xref py py-data docutils literal notranslate">AWAIT</code></a> and <a class="reference internal" href="#token.ASYNC" title="token.ASYNC"><code class="xref py py-data docutils literal notranslate">ASYNC</code></a> tokens. “async” and “await” are
now tokenized as <a class="reference internal" href="#token.NAME" title="token.NAME"><code class="xref py py-data docutils literal notranslate">NAME</code></a> tokens.
</div>
<div class="versionchanged">
Changed in version 3.8: Added <a class="reference internal" href="#token.TYPE_COMMENT" title="token.TYPE_COMMENT"><code class="xref py py-data docutils literal notranslate">TYPE_COMMENT</code></a>, <a class="reference internal" href="#token.TYPE_IGNORE" title="token.TYPE_IGNORE"><code class="xref py py-data docutils literal notranslate">TYPE_IGNORE</code></a>, <a class="reference internal" href="#token.COLONEQUAL" title="token.COLONEQUAL"><code class="xref py py-data docutils literal notranslate">COLONEQUAL</code></a>.
Added <a class="reference internal" href="#token.AWAIT" title="token.AWAIT"><code class="xref py py-data docutils literal notranslate">AWAIT</code></a> and <a class="reference internal" href="#token.ASYNC" title="token.ASYNC"><code class="xref py py-data docutils literal notranslate">ASYNC</code></a> tokens back (they’re needed
to support parsing older Python versions for <a class="reference internal" href="ast.html#ast.parse" title="ast.parse"><code class="xref py py-func docutils literal notranslate">ast.parse()</code></a> with
<code class="docutils literal notranslate">feature_version</code> set to 6 or lower).
</div>
</section>

<div class="clearer"></div>
 </div>
 </div>
 </div>
 <div class="sphinxsidebar" role="navigation" aria-label="main navigation">
 <div class="sphinxsidebarwrapper">
 <div>
 <h4>Previous topic</h4>
 <a href="symtable.html"
 title="previous chapter"><code class="xref py py-mod docutils literal notranslate">symtable</code> — Access to the compiler’s symbol tables</a>
 </div>
 <div>
 <h4>Next topic</h4>
 <a href="keyword.html"
 title="next chapter"><code class="xref py py-mod docutils literal notranslate">keyword</code> — Testing for Python keywords</a>
 </div>
 <div role="note" aria-label="source link">
 <h3>This Page</h3>
 <ul class="this-page-menu">
 <li><a href="../bugs.html">Report a Bug</a></li>
 <li>
 <a href="https://github.com/python/cpython/blob/main/Doc/library/token.rst"
 rel="nofollow">Show Source
 </a>
 </li>
 </ul>
 </div>
 </div>
<div id="sidebarbutton" title="Collapse sidebar">
«
</div>

</div>
 <div class="clearer"></div>
 </div> 
 <div class="related" role="navigation" aria-label="related navigation">
 <h3>Navigation</h3>
 <ul>
 <li class="right" style="margin-right: 10px">
 <a href="../genindex.html" title="General Index"
 >index</a></li>
 <li class="right" >
 <a href="../py-modindex.html" title="Python Module Index"
 >modules</a> |</li>
 <li class="right" >
 <a href="keyword.html" title="keyword — Testing for Python keywords"
 >next</a> |</li>
 <li class="right" >
 <a href="symtable.html" title="symtable — Access to the compiler’s symbol tables"
 >previous</a> |</li>

<li class="nav-item nav-item-1"><a href="index.html" >The Python Standard Library</a> &#187;</li>
 <li class="nav-item nav-item-2"><a href="language.html" >Python Language Services</a> &#187;</li>
 <li class="nav-item nav-item-this"><a href=""><code class="xref py py-mod docutils literal notranslate">token</code> — Constants used with Python parse trees</a></li>
 <li class="right">

<div class="inline-search" role="search">
 <form class="inline-search" action="../search.html" method="get">
 <input placeholder="Quick search" aria-label="Quick search" type="search" name="q" id="search-box" />
 <input type="submit" value="Go" />
 </form>
 </div>
 |
 </li>
 <li class="right">
<label class="theme-selector-label">
 Theme
 <select class="theme-selector" oninput="activateTheme(this.value)">
 <option value="auto" selected>Auto</option>
 <option value="light">Light</option>
 <option value="dark">Dark</option>
 </select>
</label> |</li>
 
 </ul>
 </div> 
 <div class="footer">
 &copy; 
 <a href="../copyright.html">
 
 Copyright
 
 </a>
 2001-2024, Python Software Foundation.
 
 This page is licensed under the Python Software Foundation License Version 2.
 
 Examples, recipes, and other code in the documentation are additionally licensed under the Zero Clause BSD License.
 
 
 See <a href="/license.html">History and License</a> for more information.

The Python Software Foundation is a non-profit corporation.
<a href="https://www.python.org/psf/donations/">Please donate.</a>
 
 
 Last updated on Apr 09, 2024 (13:47 UTC).
 
 <a href="/bugs.html">Found a bug</a>?

Created using <a href="https://www.sphinx-doc.org/">Sphinx</a> 7.2.6.
 </div>

</body>
</html>