diff --git a/numpydoc/numpydoc.py b/numpydoc/numpydoc.py
index 763c022d..5ddb1ca7 100644
--- a/numpydoc/numpydoc.py
+++ b/numpydoc/numpydoc.py
@@ -25,9 +25,9 @@
 import collections
 import hashlib
 
-from docutils.nodes import citation, Text
+from docutils.nodes import citation, Text, reference
 import sphinx
-from sphinx.addnodes import pending_xref, desc_content
+from sphinx.addnodes import pending_xref, desc_content, only
 
 if sphinx.__version__ < '1.0.1':
     raise RuntimeError("Sphinx 1.0.1 or newer is required")
@@ -43,7 +43,6 @@
 
 HASH_LEN = 12
 
-
 def rename_references(app, what, name, obj, options, lines):
     # decorate reference numbers so that there are no duplicates
     # these are later undecorated in the doctree, in relabel_references
@@ -89,8 +88,8 @@ def relabel_references(app, doc):
         new_text = Text(new_label)
         label_node.replace(label_node[0], new_text)
 
-        for id in citation_node['backrefs']:
-            ref = doc.ids[id]
+        for id_ in citation_node['backrefs']:
+            ref = doc.ids[id_]
             ref_text = ref[0]
 
             # Sphinx has created pending_xref nodes with [reftext] text.
@@ -103,6 +102,18 @@ def matching_pending_xref(node):
             ref.replace(ref_text, new_text.copy())
 
 
+def clean_backrefs(app, doc, docname):
+    # only::latex directive has resulted in citation backrefs without reference
+    known_ref_ids = set()
+    for ref in doc.traverse(reference, descend=True):
+        for id_ in ref['ids']:
+            known_ref_ids.add(id_)
+    for citation_node in doc.traverse(citation, descend=True):
+        # remove backrefs to non-existant refs
+        citation_node['backrefs'] = [id_ for id_ in citation_node['backrefs']
+                                     if id_ in known_ref_ids]
+
+
 DEDUPLICATION_TAG = '    !! processed by numpydoc !!'
 
 
@@ -179,6 +190,7 @@ def setup(app, get_doc_object_=get_doc_object):
     app.connect('autodoc-process-docstring', mangle_docstrings)
     app.connect('autodoc-process-signature', mangle_signature)
     app.connect('doctree-read', relabel_references)
+    app.connect('doctree-resolved', clean_backrefs)
     app.add_config_value('numpydoc_edit_link', None, False)
     app.add_config_value('numpydoc_use_plots', None, False)
     app.add_config_value('numpydoc_use_blockquotes', None, False)