From 67c8a57a45f9ca7fbb2127bf955f8b5a82b64efa Mon Sep 17 00:00:00 2001 From: "Jonathan C. McKinney" Date: Fri, 10 May 2024 22:28:55 -0700 Subject: [PATCH 1/9] Fix split and merge --- src/gpt_langchain.py | 21 +++++++++++++- src/gradio_runner.py | 9 +++--- src/utils.py | 12 ++++---- src/version.py | 2 +- tests/test_client_calls.py | 3 ++ tests/test_langchain_units.py | 53 ++++++++++++++++++++++++++--------- 6 files changed, 75 insertions(+), 25 deletions(-) diff --git a/src/gpt_langchain.py b/src/gpt_langchain.py index 786b57fa6..1067a736a 100644 --- a/src/gpt_langchain.py +++ b/src/gpt_langchain.py @@ -6789,10 +6789,21 @@ def select_docs_with_score(docs_with_score, top_k_docs, one_doc_size): class H2OCharacterTextSplitter(RecursiveCharacterTextSplitter): + def __init__( + self, + separators: Optional[List[str]] = None, + keep_separator: bool = True, + is_separator_regex: bool = False, + **kwargs: Any, + ) -> None: + """Create a new TextSplitter.""" + super().__init__(separators=separators, keep_separator=keep_separator, is_separator_regex=is_separator_regex, **kwargs) + self._separators = separators or ["\n\n", "\n", " ", " ", ""] + @classmethod def from_huggingface_tokenizer(cls, tokenizer: Any, **kwargs: Any) -> TextSplitter: def _huggingface_tokenizer_length(text: str) -> int: - return get_token_count(text, tokenizer) + return get_token_count(text, tokenizer, add_special_tokens=False) return cls(length_function=_huggingface_tokenizer_length, **kwargs) @@ -6801,6 +6812,8 @@ def split_merge_docs(docs_with_score, tokenizer=None, max_input_tokens=None, doc joiner=docs_joiner_default, non_doc_prompt='', do_split=True, + do_first_semantic_split=False, + hf_embedding_model=None, verbose=False): # NOTE: Could use joiner=\n\n, but if PDF and continues, might want just full continue with joiner='' # NOTE: assume max_input_tokens already processed if was -1 and accounts for model_max_len and is per-llm call @@ -6817,6 +6830,11 @@ def split_merge_docs(docs_with_score, tokenizer=None, max_input_tokens=None, doc do_split &= any([x > max_input_tokens for x in tokens_before_split]) if do_split: + if do_first_semantic_split and hf_embedding_model is not None and 'model' in hf_embedding_model: + from langchain_experimental.text_splitter import SemanticChunker + text_splitter = SemanticChunker(hf_embedding_model['model']) + docs_with_score = text_splitter.create_documents(docs_with_score) + if verbose: print('tokens_before_split=%s' % tokens_before_split, flush=True) @@ -8009,6 +8027,7 @@ def get_chain(query=None, docs_token_handling=docs_token_handling, joiner=docs_joiner if not doing_grounding else "Document xx", non_doc_prompt=estimated_full_prompt, + hf_embedding_model=hf_embedding_model, verbose=verbose) # in case docs_with_score grew due to splitting, limit again by top_k_docs if top_k_docs > 0: diff --git a/src/gradio_runner.py b/src/gradio_runner.py index 358a64aff..bc31e38ed 100644 --- a/src/gradio_runner.py +++ b/src/gradio_runner.py @@ -5376,11 +5376,11 @@ def larger_str(x, y): text_output], outputs=[text_output, chat_exception_text, speech_bot], ) - retry_user_args = dict(fn=functools.partial(user, retry=True), + retry_user_args = dict(fn=functools.partial(user, retry=True, sanitize_user_prompt=kwargs['sanitize_user_prompt']), inputs=inputs_list + [text_output], outputs=text_output, ) - undo_user_args = dict(fn=functools.partial(user, undo=True), + undo_user_args = dict(fn=functools.partial(user, undo=True, sanitize_user_prompt=kwargs['sanitize_user_prompt']), inputs=inputs_list + [text_output], outputs=text_output, ) @@ -5566,10 +5566,11 @@ def clear_all(): inputs=[my_db_state, requests_state, guest_name, retry_btn, retry_btn], outputs=[my_db_state, requests_state, retry_btn], queue=queue) - submit_event3a = submit_event31.then(**user_args, api_name='retry' if allow_api else False) + submit_event3a = submit_event31.then(**retry_user_args, + api_name='retry' if allow_api else False) # if retry, no longer the saved chat submit_event3a2 = submit_event3a.then(deselect_radio_chats, inputs=None, outputs=radio_chats, queue=queue) - submit_event3b = submit_event3a2.then(**user_args2, api_name='retry2' if allow_api else False) + submit_event3b = submit_event3a2.then(**retry_user_args2, api_name='retry2' if allow_api else False) submit_event3c = submit_event3b.then(clear_instruct, None, instruction) \ .then(clear_instruct, None, iinput) submit_event3d = submit_event3c.then(**retry_bot_args, api_name='retry_bot' if allow_api else False, diff --git a/src/utils.py b/src/utils.py index 787e7dd92..89b16d663 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1993,14 +1993,15 @@ def str_to_dict(x): return x -def get_token_count(x, tokenizer, token_count_fun=None): +def get_token_count(x, tokenizer, token_count_fun=None, add_special_tokens=True): # NOTE: Somewhat duplicates H2OTextGenerationPipeline.get_token_count() # handle ambiguity in if get dict or list + other_kwargs = dict(add_special_tokens=add_special_tokens) if hasattr(tokenizer, 'add_special_tokens') else {} if tokenizer is not None: if hasattr(tokenizer, 'encode'): - tokens = tokenizer.encode(x) + tokens = tokenizer.encode(x, **other_kwargs) else: - tokens = tokenizer(x) + tokens = tokenizer(x, **other_kwargs) if isinstance(tokens, dict) and 'input_ids' in tokens: tokens = tokens['input_ids'] if isinstance(tokens, list): @@ -2013,7 +2014,8 @@ def get_token_count(x, tokenizer, token_count_fun=None): raise RuntimeError("Cannot handle tokens: %s" % tokens) elif token_count_fun is not None: assert callable(token_count_fun) - n_tokens = token_count_fun(x) + other_kwargs = dict(add_special_tokens=add_special_tokens) if hasattr(token_count_fun, 'add_special_tokens') else {} + n_tokens = token_count_fun(x, **other_kwargs) else: tokenizer = FakeTokenizer() n_tokens = tokenizer.num_tokens_from_string(x) @@ -2341,7 +2343,7 @@ def get_docs_tokens(tokenizer, text_context_list=[], max_input_tokens=None, docs assert max_input_tokens is not None, "Must set max_input_tokens" tokens = [get_token_count(x + docs_joiner, tokenizer) for x in text_context_list] tokens_cumsum = np.cumsum(tokens) - where_res = np.where(tokens_cumsum < max_input_tokens)[0] + where_res = np.where(tokens_cumsum <= max_input_tokens)[0] # if below condition fails, then keep top_k_docs=-1 and trigger special handling next if where_res.shape[0] > 0: top_k_docs = 1 + where_res[-1] diff --git a/src/version.py b/src/version.py index 29a26c02b..f2d4dfb07 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1 @@ -__version__ = "4ad44589074016621f961aa89582f0b85f54f76b" +__version__ = "1ad4c7e4622c4112c323bee41183162933927342" diff --git a/tests/test_client_calls.py b/tests/test_client_calls.py index 9783c91ac..02a6822b5 100644 --- a/tests/test_client_calls.py +++ b/tests/test_client_calls.py @@ -3124,6 +3124,9 @@ def test_client_chat_stream_langchain_fake_embeddings(data_kind, base_model, inf '| 0 | 1 | 2 |\n|:---------|:--------------------------------------------------------------------------------------|----:|\n| Item 7. | Management’s Discussion and Analysis of Financial Condition and Results of Operations | 8 |\n| Item 7A. | Quantitative and Qualitative Disclosures About Market Risk | 15 |\n| Item 8. | Financial Statements and Supplementary Data | 16 |\n| | Report of Independent Registered Public Accounting Firm | 16 |\n| | Consolidated Statements of Earnings | 17 |\n| | Consolidated Statements of Comprehensive Income | 17 |\n| | Consolidated Balance Sheets | 18 |\n| | Consolidated Statements of Shareholders’ Equity | 19 |\n| | Consolidated Statements of Cash Flows | 20 |\n| | Notes to Consolidated Financial Statements | 21 |\n| Item 9. | Changes in and Disagreements With Accountants on Accounting and Financial Disclosure | 33 |'] +texts_long = ["""You cannot play any games about this. You have to admit that this is wrong. I think especially for mathematicians to come in and see an environment where there's guiding ideas that people haven't really worked out, and a lot of things are known, do not work for known reasons. But people are still acting as if this is not true and trying to figure out how to do something and make career for themselves. Peter Wojt is a theoretical physicist and a mathematician at Columbia University. He's been an influential figure in the ongoing debates surrounding string theory. His critiques, as articulated in his book, Not Even Wrong, strike at the heart of many popular assertions about this framework. Professor Hoyt also has a widely read blog in the math and physics scene called Not Even Wrong, so it's the same name. And the links to all resources everything mentioned will be in the description as usual. take meticulous time stamps and we take meticulous show notes in one sense the problem with string theory is the opposite of the problem of fossil fuels with fossil fuel companies You have a goal let's say it's to wash your clothes and you're able to achieve that goal but you produce negative externalities where a string theory has plenty of positive externalities but arguably achieves little toward its initial goal Professor White introduces a novel tow approach called Euclidean Twister unification. You may recognize that term Twister, as it's primarily associated with Roger Penrose. Twisters provide an alternative to spacetime descriptions in quantum physics. Peter's application of Twisters is in the Euclidean setting, and he talks about how this significantly changes the playing field. It opens up a connection between gravity and the weak interaction, because space-time in this formulation is inherently Cairo. We also talk about spinners and Michael Atiyah. You know how some people are Christian mystics or Muslim mystics? Well, Atiyah seems to be a spinner mystic. We alternate between technical and more intuitive discourse. If you're new to the theories of everything channel, this is par for the course, and my name is Kurt Jai Mungle. Usually what we do is we interweave between rigorous, steep technicality, and then periods of explaining the intuition behind what was just said. In other words, you can think of it as high intensity interval training for the mind. Recall the system here on Toe, which is if you have a question for any of the guests, whether this guest or from a different Toll podcast, you can leave a comment on that podcast with the word query and a colon. This way, when I'm searching for the next part with this guest, I can press Control F, easily finding it in the YouTube studio backend. Further, if I'm able to pose your query, I'll cite your name verbally, either aloud or in the description. Welcome and enjoy this episode with Peter White. Welcome, Professor. Thank you so much. It's an honor to have you. I've been wanting to speak to you for almost two years since you came out with Euclidean Twister Theory or Euclidean Unification Theory. And while here you are. Well, thanks. Thanks for having me on. I'm looking forward to the opportunity to kind of be able to talk about some of these topics. And I've certainly enjoyed some of your other programs. And the one with my friend Edward Frankel recently was really spectacular. Thank you. Yeah, that's all due to Ed, of course. Okay. What are you working on these days? What's your research interests? Yeah, so there's something very specific. I'm just in the middle of trying to finish a short paper about an idea, which I'm not quite sure what they're... I guess I've for now entitled, the draft of the paper is titled Space Time is Right-Handed. And there's a slight danger that I'll change conventions. It'll end up being that slight space time is left-handed, but I think it will stay right-handed. And that's related to the twister stuff that I've been working on for the last few years, which I'm still quite excited about. But there's something at the, there's one kind of basic claim at the bottom of what I'm trying to do with the twisters, which is, I think to the standard way of thinking about particle physics and general relativity and spinners, it's initially not very plausible. I should say one reason that I actually didn't, it took me a long time to get back to the Euclian twister stuff from some early ideas years ago was that I didn't actually believe that this basic thing that I needed to happen and could happen. And I think lots of other people have had the same problem with this. And the more I looked into the twister stuff, the more I became convinced that something like this had to work out. But more recently, the last few months, I've come up with an understanding in much simpler terms, not involving twisters, just involving spinners, about the really unusual thing that's going on here. And I think that, you know, I've been trying to write up kind of an explanation of the basic idea. And I think it's a fairly simple one. And as I've been writing it up, I keep thinking, well, wait a minute, can this really work? There's no way this can actually really work. But the more I've been thinking about it, the more I've been convinced, yes, this actually does really work. So I'm hoping within the next few days to have a final version of that paper, well, not a final version of that paper I can at least send around to people and try to get comments on and also read about it and publicly on my blog. I read the paper. Thank you for sending you. Yeah, what you have is a very, it was a very early draft of it, which made even less, hopefully the, I'll have something that will make more sense will be what the public will see, but we'll see. Yeah. Do you think spinners are more simplified or easy to understand the twisters? Oh, yeah, yeah. So spinners are really very basic, very, very basic things. I mean, every elementary particle like electrons are the way you describe them. They're spinners. They're going to have nature as spinners. You have to electron wave functions are spinners. And so they're in every, you know, every physics every if you do quantum mechanics or do quantum field theory you have to spend a fair amount of times at spinners so spinners are very very basic things and they're not um i spent a lot of my career kind of thinking about them trying to better understand them and i keep learning new things and it's in the last few months i kind of i something about them, which I think is new, at least I've never seen before. And this is what I'm trying to write about it. But they're very fine metal objects. It's a little bit hard to, anyway, I can give you a whole lecture on spinners. I'm not sure how much of that you want or where you want to start with that. Right. Well, there's one view that we can understand them in quotes algebraically, but that doesn't mean we understand what spinners are. So that's the Michael Latia approach where he says it's like the letter I, the complex eye, the imaginary I, back in the 1400s or 1500s. It's only now or a couple hundred years later, you realize what they are. And so sure, we have many different ways of describing spinners mathematically, but it's still a mystery as to what they are. So do you feel like, no, we understand what they are, or there's much more to be understood more than the formalism? Well, yeah, it's very interesting. You bring up Atia, yeah. So Atia at various points, did make this argument that there's something very interesting in which we don't understand going on of the spinners and that yeah he i think was thinking of it in a much more general context spinners you know are really if you try and do geometry of any kind um or reminding in geometry you re expressing everything in terms of spinners instead of in terms of vectors and tensors gives you a very different, in some ways, more powerful formalism, but one that people are not that used to. And it has some amazing properties. It's kind of deeply related to notions about topology and K-theory and the Daraq operator gets into it. So the thing that made attia you know really most famous his index there was singer you know this is that it's basically saying you know you can compete everything comes down to a certain kind of fundamental case and that is the final case of the drach operator and spinners so he was seeing spinners kind of at the, you know, as this really kind of central thing to the most important thing that he'd worked on. And so there's a lot to say. So there's a lot known about spinners, but there's also a lot, it's a little bit mysterious where they come from. I think the new stuff that I've been more, so I've been thinking about that a lot over the years, but the new stuff that has gotten, where I think there's something new that I see going on is not the general story about spinners, but a very, very specific story about spinners in four dimensions. So you have spinners in any dimension. Any dimension, you can write down spinners and they're useful. But in four dimensions, some very, very special things happen. And the other very, very special thing, it's interesting thing that's going on in four dimensions is that from the point of view of physics, there's two different signatures that you're interested in. You're interested in either spinners in the usual kind of four dimensions where all four dimensions are the same and you're just trying to do Euclidean geometry in four dimensions, which I might sometimes call Euclidean spinners, or you're interested in spinners of the sort that you actually observe in a relativistic quantum field theories where the geometry is that of Minkowski space. So sometimes we refer those as Minkowski spinners. And so you have two different versions of four dimensions, one with a totally positive signature and one where one direction has the opposite sign than the others in the metric. So you have to treat time differently than space, and that's Minkowski space. So there's two different things than the general story that I'm interested in here. One is very specific, what has specifically the geometry of four dimensions, and the other is very specifically the relation between Euclidean and Minkowski signature spinners. So is it your understanding or your proposal that the world is actually Euclidean, and it's been a mistake to do physics in a Minkowski way? When we wick rotate, we see that as a mathematical trick. And you're saying, no, no, no, that's actually the real space. That's the real, quote unquote, even though there's something imaginary about it. And the Minkowski case was the mistake. Like, an analogy would be, we operate in U.S. USD. And then for some calculations, it's easier to go into yen. And we think that the actual world is operating in the United States, and the calculations are just something to make the numbers easier. And then you're saying, no, no, no, what's really happening is in Japan, and it's been a mistake to go into the USDA, or the USD is just to make the math easier. So is that what you're saying or no? Well, so this goes back more to the Euclidean twister stuff. Yeah. So there, well, yeah, it's been well known in physics that you really kind of, that the problem in there's a problem with Minkowski space time. If you try and write down your theory in Mkowski space time, you, the simplest story about how a free particle evolves, you write down the formulas for what's a free particle going to do, what's its propagator, and you see that it's just ill-defined. There is no, you know, you've written down a formula which mathematically is ill-defined. It needs more information in order to actually be a well-defined formula. And I mean, technically, if you look at any physics book, you'll see they're saying, well, you know, we're going to do, the answer is this integral, and you look at this integral, and this integral is going straight through two poles, poles, and that's just ambiguous. You don't know how to define such an ambiguities about how you define such an rules. So the one, the aspect, you've always known you have to do something like with rotation. You have to do something. You have to get rid of those ambiguities. And one way of getting rid of those ambiguities is, you know, analytically continuing and making time a complex variable, analytically continuing it, analytically continuing maybe to Euclidean signature, and there the formulas are well defined. So it's, yeah, I'm not sure, I'm very comfortable saying one of these is real and one of these is not. It's the same, it's the same formula. It's just you have to realize that to make sense of it, you have to kind of go into the complex plane in time. And you can, if you things are analytic, if this is a holomorphic function in time, you can either evaluate what happens at imaginary time or you can make time real, but you have to take the limit in a certain way, moving, like perhaps starting with imaginary time and then moving analytically continuing a certain direction to get real time. But that's the standard story. That's not me saying this. That's a standard story. Right. And then there's a, how do you, what sense do you make of this? Is this just a mathematical truck, which a lot of physicists will say, well, that's just some kind of weird mathematical trick. It's not, has nothing to the reality. Or do you take this more seriously? So what's always fascinated me is more is that it's fairly clear what's going on if you just talk about scalar fields. If you talk about particles with spin zero or fields that transform trivially under rotations, you know, what happens when you go to imaginary time is, you know, it's quite interesting and in some ways tricky, but it's very well understood. But it's never actually been very well understood. What happens when you have spinner fields? And this is the problem is that these spinners in Euclidean signature and spinners in a calcium signature are quite different things. And so you can't just say, oh, I'm going to analytically continue from one to the other because they're not related. Anyway, it's very unclear how you're going to do that. And so there's also a similar story in Twister theory. You can do Twister Theory, Yonkowski Space Time, which is what Penrose and his collaborators mostly did. Or you can do it in Euclidean signature space time, which is what Atia and a lot of other people and mathematicians have done. And in principle, the two are related by analytic continuation. But the way that works is quite, you know, I think it's much subtler than you expect. And so what I've been interested in, you know, most recently this business about, it really is a claim that the standard way of thinking about how you analytically continue between these two different kinds of spinners is you're making kind of a wrong choice when you do that. And there's a good reason for the standard choice you're making when you normally when you do that. But there is actually another choice you can make, which is that instead of working with spinners which are kind of symmetric, there's two different kinds, which by convention you can call right and left-handed or positive and negative chirality. And the standard setup treats this question, you know, symmetrically, but between the plus and minus, the chirality is between right and left spinners. But it's, what I've kind of realized recently is it looks like it's quite possible to make this setup completely asymmetric so that you just describe spinners using these right-handed or positive chirality spinners. You just don't use the left-handed ones at all in your construction of space time. You can do that. It appears to be, and that's why this paper is called space time is right-handed. Is it the case that you could have called it space-time as chiral, and you could have equivalently described as left-handed, or is there something specific about right-handedness? No, yeah, yeah. It's certainly, it's a matter of convention, which, but you base have but you basically, to say it a little bit more technically, you know, the, the, the, the, the, the, the, the, the, the, the, the, the, the, the, the, the, the, the, the, the, the, the, the, the, the, if you're, is this group called SL2C, it's two by two complex matrices, a determinant one. Um, and what you realize is when you, if you work, if you come, if you work in complex version of four dimensions, the symmetry group is two copies of SL2C. And you can call it a plus copy and a minus copy or you can call it a right copy and a left copy, but there's two of them. And the standard convention in order to get analytic continuation to work out the way people expected has been to say that the physical Lorentz group that corresponds to our real world is not chiral asymmetric. It's kind of a diagonal, which is you use both the right and left. And you have to complex conjugate when you go from one side to the other. But it kind of the Lorentz group, the SL2C Lorentz group we know, is supposed to sit as kind of a diagonal thing, which is both right, right, and left. But what I'm kind of arguing is that, no, you can actually set things up so that the, the Lawrence group is just one of these two factors. It could have been the right factor, left factor. You have to make your choice of convention. But so it is very much a chiral setup. But you only, the strange thing about this is you only really see this when you complexify. If you just look at Minkowski space time, you know, you don't actually see this, anyway, you don't see this problem or you don't see this ability to make this distinction. It's only when you go to Euclidean space time where the rotation group really does split into two completely distinct right and left things. Or if you go to a complexified space time where you have this two copies of SL2C, it's only in those contexts that you actually see that there is a difference between choosing the diagonal and choosing the right-handed side. So for SL2C, you call that the Lorentz Group. Is that technically the double cover of the Lawrence Group? Yeah, people use both terminology. If you're going to work with spinners, you have to use a double cover. But yes, it's also, yeah, yeah. Sometimes you might want to say that S.O3 is the Lorentz group and this is the double cover. But most of you're working, you're interested in doing working with spinners, and then you have to use the double cover, really. Yes, yes. So is there a reason that triple covers or quadruple covers aren't talked about much? Is it just because of experiment? There's nothing there. Well, it's more than mathematics that they don't. There is, I mean, there is, the rotation groups of any kind, you know, have this, have this twofold nature. There is this spin double cut. There is this, they have these spin double covers. In many cases, you can kind, one way of seeing this is just a basic topology, the topology of rotations has a, you know, has a plus and minus thing in it, which you kind of, and you have to do something about that. So there aren't any kind of known, mathematically interesting, triple covers, etc. Now, in the standard model, the way that it's written in bundle language is that it's a principal bundle, and then the gauge groups are the structure groups. And then for general relativity, you have a tangent bundle. And then some people say that the gauge group of GR is the dipheomorphism group. But is there a way of making that into a bundle, like a principal bundle with the diphomorphism group? How is one supposed to understand that as a bundle construction? Yeah, yeah. Anyway, there's a lot of different ways. There's several different ways of thinking about geometry and about Romanian geometry. And yeah, and this starts to get a complicated subject. But maybe the best way to, well, thinking in terms of different amorphism groups is something you can do. It's actually not my favorite way of doing this kind of geometry. And for the reason is that it, maybe let me just say something about an alternate way of thinking about geometry, which seems to me more powerful. Maybe actually to motivate this a little bit better. If you just think about diffamorphism groups, it's very, very hard to understand what spinners are and where they come from. You really kind of can't see them at all if you're just thinking about the diphthomorph group of a manifold. So the the other formulation of geometry going back to Carton, which makes it much, makes it much easy to see where spinners are going on going and is a lot more powerful in other respects, is to think not about your not about a manifold, but about a bigger space, which is a bundle that for each point in the manifold, you consider all possible bases for the tangent bundle. It's also called frames. And so this is sometimes called the frame bundle. And so it's kind of saying if you want to understand geometry, you should look at the points of space and time. But at the same point, you also got to think about the tangent space and you should think about the possible bases of the tangent space and the so-called frames. So you should always kind of think, instead of writing all your formulas in terms of local coordinates on the manifold, you should think about your problem as being a problem that lives up on the frame bundle and that you always, you're not just, you're not just at a point to space time, but you've also got a frame. And then, but then you have to be careful to kind of work kind of equivariantly that you have, you know, you can change your choice of frame. You can rotate your frames. So you have, you kind of work up in the frame bundle, but equivariantly with respect to rotations or whatever. So that's, that gives a lot more structure to the problem. In particular, it allows you to easily say what spinners are, which you couldn't if you just talked about. So, so anyway, there's a lot more one could say about Diffey Morphor's in groups and that, but just in terms of the relation to the spinner stuff, maybe it's just to forget about it. Just to say it that way. It's not, you have to do something quite different if you're going to talk about spinner. Right. Okay, now the problem you were working on earlier that you said you weren't sure if it would have a solution and you're finding that it does what was it in the early part of the conversation which you were working on your research interests well do you mean right at the beginning where i'm still what i'm still confused about yeah okay but it seemed to me that you were saying you're solving the problem. Oh, this. Yeah. So this was, I mean, this was actually, it goes back to when I was graduate student or postdoc, it was first occurred to me. You know, actually maybe to kind of explain how this all came about. So I was a graduate student in Princeton and I was working on lattice gauge theory. So we're working on this kind of formulation of Yang-Mill's theory on a lattice. And so you could actually do computer calculations of it. And so I was trying to understand, you know, there's a lot of interest in topological effects in Yang-Mills theory. And I was trying to understand how to study those in the kind of numerical calculations on the lattice. And then, so I made some progress on that. But then the next thing that really occurred to me was exactly as spinners came up. It's like, besides having Yang Mills theory on a lattice, we also want to put spinner fields on the lattice. So there's this really beautiful way of putting gauge fields in the lattice, the Yang Mills theory, which kind of respects the geometric nature of the gauge fields very, very nicely. It's kind of the Wilson's lattice gauge theory. But there isn't, if you try and put spinners in the lattice, a lot of very mysterious things happen. And again, in some sense, the problem is that if you're just looking at this lattice that you've written down, it's clear kind of what the discrete analogs of vectors are and of planes and of those things. But it's very, very unclear what, since you don't really have a good way of thinking about spinners in terms of kind of standard geometry of, you know, lines, planes, et cetera, you don't really know how to put the spinners on a lattice in a way that respects their geometry. And if you try to write down the formulas or do anything, you run into a lot of weird problems. There's a lot of, anyway, there's a long story about what happens if you can try with spinners and lattice. Is this related to doubling, like the species doubling? Yeah, so there's one, yeah, so one thing you find is that you really, there's no kind of consistent way to put kind of a single kind of Fermion in the lattice, that if you try and do it, any way you know of doing it kind of produces all these kind of extra versions of the same thing and you have to somehow disentangle those. That's part of the problem. Okay. But that's when I started thinking about the geometry of spinners and some ideas about putting them on the lattice. And then what I was seeing, I started to see that, wait a minute, you know, if you, so this is all happening in Euclidean space where the rotation group is a copy of two SU2 u s u.u2s there's again a left-handed one or a right-handed one if you like and um what i was seeing really was that the some of the choices of it the geometry is trying to use to put these things in the lattice gave me kind of things occurring and kind of multiplets that that look that had the same SU2 structure as what you see in a generation of electro weak particles so in a generation of electro week um like to be particles that you for instance have you have a neutrino left end of neutrinos and you have right-handed left-hand electrons for instance. And those have certain transformation properties under the SU2 and under a U-1. And those were the same ones that I was seeing when I was trying to construct these spinners. So I had the, so it seemed to me, if you can think of part of this rotation group, this SU2, as an internal symmetry, as the symmetry of the weak interactions of the Weinberg's Sala model, then you could actually, anyway, you got all sorts of interesting things to happen. But the thing that this, but making this idea work really required that some explanation of why in Euclidean space, what you thought were spacetime symmetries that really broke up into half space time symmetries and half an internal, internal symmetries, which didn't affect space time. So I never, this is what for many years after looking at this, it was like, well, this just can't work. I mean, you can't, if you just look at the whole formalism for how you've set this up and, you know, both of these SC2s have to be space time temperatures. You can't, they're both going to affect space time. You can't, you can't get away from that. Other people didn't see this as a problem? No, no, I think everybody saw this as a problem. I mean, I think anybody who ever looked at this idea of trying to get, you know, one of the, part of the four-dimensional rotation symmetry to be an internal symmetry has probably backed away, backed away from it for the same reason, saying, well, wait a minute, this can't, you know, I just can't see how that could actually happen, that you have to, you're telling me this should be an internal subject which doesn't affect space time, but it looks to me that you're rotating space time with it, so you can't do that. And so this is what, for many years, kind of kept me from going back to that, from those ideas. And as I learned more about quantum filter, actually, one motivation, as I was teaching this course on quantum filtering, quantum filtering in the back of my mind is, okay, you know, as I go along and teach this course, I may not explain this to the students, but I'm going to very, very carefully look at the formalism and I'm going to understand exactly how this analytic continuation is working of these spinners. And I'm going to, you know, and I'm going to see that you know it looks like this has to work and I'll finally understand why and then I can stop thinking about this but but I kind of as I was teaching this as I was looking at this I kind of never actually saw you know anyway I never actually really saw the argument for why this why this be a space side of symmetry. It looked like it had to, but you couldn't quite pin down why. Anyway, so then when I went back to the twister stuff, I became convinced that if you think about everything in terms of twisters, then the whole twister setup is naturally, chirally, asymmetric. So you kind of, from the twister point of view, this kind of thing looked a lot more plausible, and I got more interested in it again. But it's only very recently, the last few weeks, the last couple months, that I've kind of, I kind of have a very good understanding of exactly why it seemed that, you know, that what I, that why I was right, that this should be impossible. There is a standard assumption that you're making, which makes what I wanted to do impossible. But it's also possible to not make that assumption and do something else. And that assumption is? It's the symmetry between right and left. It's kind of when you go between Minkowski and Euclidean spinners, you know, the setup that you use to analytically continue, do you do that in a setup which is right-left symmetric? And if you want the setup to be holomorphic then you have to use the right left symmetric one but what it's so simultaneously i realize yes you can yeah yes i mean this in the standards there was a very very good reason that i and everyone is skeptical that this can make sense but there there actually is a way around it. You can just decide, okay, I'm going to, I'm going to just use right-handed spinners, and I'm going to, and you can get a theory that makes sense. I don't know if I'm jumping ahead, but I recall in one of the lectures that I saw online of you, and you were giving the lecture, I believe Cole Fury was in the audience, you were saying that what we have to use are hyper functions. Yeah. Am I jumping ahead because you're saying, no, no, it's not going to be holomorphic? No, but actually hyperfunctions are really part of the holomorphic story. They're, yeah, they're not, they're, I mean, hyper functions are really just saying, so so what I was saying when I was trying to explain this business about you know why about WIC rotation and that and that things were that if you write down the standard formulas you end up with something in a Kasek space on which is ill-defined okay and then you have to use it via Rick Rotation or analytic continuation. There's just another way of saying that more, putting in a more interesting mathematical context, is to say that the things that you're looking at in Kowski-space time are not actually normal functions. They really are what am I? They are best thought of as hyper functions. In this case, they're hyper functions, which are just, um, analytic, which are just kind of bound, boundary values of analytic things as you, uh, approach, uh, approach the real line. But, um, yes, so the hyper function story is just kind of part of the standard. It's really part of the weird rotation story. Yeah. Okay. Yeah. But what I'm, I mean, this latest thing I'm trying to do actually gets away from analytic continuation. You're not, you really, I'm really, I'm still kind of, you know, trying to wrap my head around exactly what the implications of this are. But you are, you're not doing the standard sort of analytical continuation anymore. The standard sort of way of analytically continuing, which uses all four space time dimensions, that you're not, you're not doing that. You're doing something, something different and it's unclear. Anyway, I mean, if you start writing out formulas, you'll still get the same story with hyper functions. What prompted you to then go look at twisters? And by the way, is it called a twister formalism or twister formulation? I don't know. Either one is... I don't know if those are used interchangeably. Because I hear, for instance, that there's different quantum formalisms like Vigners or interaction or path or categorical. But then sometimes I hear, yeah, the categorical formulation of quantum mechanics. I'm like, okay, you get the idea. Well, they're not, I mean, the thing about Twisters is they're not actually, I mean, maybe a good thing to say about Twisters is, we don't actually know exactly what their relevance is to the real world. So you might, if you had a, if you have a well-developed idea using Twisters for describing the real world and you wanted to contrast it to other similar descriptions, you might want to say, oh, this is the Twister formalism or maybe Twister formulation. I don't know. But it's a little bit, but either one is a little bit premature in terms of physics, that we don't actually know exactly how the twisters are related to the real world. So it's not like you can translate a real world problem to twister formalism and then back? Well, you can, so twisters, maybe... So twisters are a bit like spinners, but the, um, so they have some of the mathematical properties of spinners, but, but they do something more interesting. They're kind of a higher dimensional thing. Maybe one of the best things to say about them is that they're, um, they're very useful. If you want, if you, so if you want to understand Minkowski space time, you know, you, this is what Einstein figured out. You can, you can use Minkowski's geometry, Minkowski metric, if you want to talk about just vectors and metrics and tensors, or if you talk about Mokowski space type spinners, if you want, and that's what I've been most interested in. But the other interesting thing about our theory is when we write them down in Mikowski space time. Theories of mass of massless fields and things like Yang Mill's theory, they have this bigger invariance group than just under rotations and translations. They're conformally invariant. So the geometry of chrysters really comes into its own, if you're trying to describe to understand the properties of space time under conformal transformations. And anyway, so that's kind of a motivation. So if you don't care about conformal transformations, you may not be very interested in spinners, but if you really want to understand, you know, what is, how do I write down my theories and how do I have a version of you of Metcowski space time that, where the conformal group acts in a nice linear fashion and where everything works out. And the spinner, now you can call it a formalism or formulation, but it's a way of doing conformal geometry. It really comes into its own. So that's, so spinners, you know, go, I mean, twisters go way, way back. And, you know, this really was mainly Roger Penrose is doing in the 60s. And, you know, and he was very interested in using them to understand, you know, things happening in Minkowski space time and especially the conformal invariance of these things. And so there's a huge amount of effort and a lot of beautiful things discovered during the 70s, especially by him and his collaborators in Kowski space time. And then Atiyah realized that you could take this over and do some very, very interesting things in Ramanian geometry and Euclidean space time. Yeah. So, I mean, I was, you know, I kind of learned about this geometry at raise points. That sentence could be said about Atea in the most general form. And then Atia realized you could use this for underscore with geometry. Yeah, yeah. Yeah. So anyway, so I've been kind of aware about Twisters for a long time, but I, you know, I didn't see. Anyway, I actually wrote a very speculative paper long, long ago about this. And it mentioned the connection to twisters, but there's just a lot about them that, you know, I didn't understand back then. It took me many years to understand. And especially the relationship between is Euclidean signature and Minkasee signature spinners, how they're related is. That's quite a tricky story, which takes me a long time and understand. So you have the splinter in your thumb for decades about the spacetime symmetries and them acting not just on spacetime. What happened in 2020 and 2021? I'm trying to think. Now, I'm trying to think what specific one thing had happened in 2020 was COVID so right in your mind what happened so 2019 then no no no but this is actually relevant because actually in 2020 I was much more and I was thinking of this stuff but yeah but yeah but in 2020 all of a sudden you're kind of you know you're at home you're at home that you're just sitting there and uh i all the opposite home and i don't have a lot of all the usual distractions or whatever and so and so that actually um i actually gave me some of the more time to kind of think peacefully about uh about some of this stuff and make some progress yeah so i'd have i'd have to i mean i kind remember now that, you know, exactly which things became clear at which times. But it's been a slow, it was a slow process of various things clarifying. But I think maybe that was one of the main things, is to finally get a picture in mind of how Euclidean and Minkowski twist your theory all fit together. Awesome. How does it fit? Is there a way of explaining it? Well, I mean, maybe the best thing to say about twister theory is that it really kind of naturally wants to be a theory of complex space time. And this is the thing. If you write, if you say, I'm going to study four-dimensional complex space time and I'm interested in its conformal group and things like that, then the Twister story is actually very, very simple. It's very, I mean, you're basically just saying that there's a four-complexed dimensional space and a point in space time is a complex two plane in that four-dimensional space. So points, anyway, yeah, so instead of thinking of the way of normal thinking of some space with these points, well, you've got to think about, just think about the complex two planes and complex four-dimensional space, and, you know, everything just kind of drops out of that. And there is one, there's a beautiful relation of that story to the theory of spinners, is that, and this is kind of the relationship between the theory of twister and theory of spinners. In twister theory, a point in four-dimensional space-time is a complex two plane. That by definition of what a point is. And now that, but that complex two plane. That's the definition of what a point is. But that complex two plane, that kind of tautologically answers the question of where do these spinners come from? Because the space of spinners is a complex two plane. Well, you know, so from the standard point of view, it's like, you know, as I was saying, if you just think about the diphthomorphism, it's very, very hard to even say what a spinner is. So where are these weird complex two planes coming from? Well, from the point of view of twister theory, it's purely tautological. It's just, you know, the two plane is a point. So the spinner, the spin one-half two-plane, complex two-plane is describing the spin of a of an electron is exactly a point anyway that that's exactly what what the definition of a point is so you can't a point in twister space or a point in space time spilling in space time yeah so as twister space is a four thing. And but the points in it, and so the points in it correspond to various structures in space-time, but the complex two planes in it correspond to the points in space-time. Anyway, that's one of the basic. Yeah. So then is the statement that the points in spacetime are the same as spinners or the points in space- or the structure of space time gives rise to the structure of spinners and vice versa or are none of those statements correct? I think, yeah, I know, I think both of them. I mean, it really is telling you, Twister theory is really telling you that it's a way of thinking about space time in which... And sorry, this is four dimensional space time. Four dimensional space time, yeah, yeah, yeah. It's a way of thinking about, yeah. So, Twister theory is very, very special to four-dimensions. It doesn't really is, it's a way of thinking about space-time in which, you know, the occurrence of spinners and their properties are just completely tautological. They're just built into the very definitions. Sociologically, why do you think it is that Penrose's Twister program, firstly, has been allowed to continue because many other programs just die out if you're not loop or string or causal or asymptotic. Like there's just four as far as I can tell. Five with Penrose. So why is it alive, and then why hasn't it caught on? Well, for, I mean... Or maybe you disagree. It's not alive. No, no, no. It's very much alive. It's very much alive. And still... And so there's an interesting kind of history. But a lot of it was really... So he had this idea, and he's raised places as explaining how he came up with it. And he was very, very struck by this. And, you know, so he quite successfully at Oxford built up a group of people working on this. And so, you know, it was a good example of kind of how normal science kind of works sociologically. You know, somebody comes up with a good idea and they actually build a group of people around them and people do, as people do more work, they learn more interesting things about this more people get interested so you know he always you know throughout the 70s I would say into the 80s there always was a quite healthy group of people you know working on pedros or people somehow having some relation to penrose collaborators were working on this so it was um anyways but perfectly but perfectly normal science. It wasn't so clear, though, how to get, it was clear, some things were very clear, some things were clear that this was really a beautiful way of writing down conformally invariant way of equations and studying their properties. So there were, there was, the beauty of the idea and the power to do certain things was known. But it didn't seem to be necessary or have any particular connection to specific problems in particle physics. So particle physicists would look at this and say, well, that's nice, but, you know, I don't, that doesn't actually tell me anything. You know, if I needed to do some conformally invariant calculations, I might be able to use that, but it's not actually telling me something really that, you know, really knew I can't get elsewhere. And then, you know, and then in the 80s, you also had, you know, Atea got into the game, and there's a lot of mathematicians got into it through the, the relations to the, on the Euclidean side. So, you know, it was, you know, especially among mathematicians, mathematical physicists, it was a fairly, it remained a very active area, and it still is to this day, you know. A lot of it was based in Oxford, but also a lot of other places. But yeah, I think the, but in terms of its implications for physics, you know, I would say the thing that to me is, I think Penrose and his people trying to connect this to physics in an interesting way, they kind of ran into, anyway, they kind of ran out of new ideas. There are some things that they could do, but they couldn't actually get any kind of a really killer app, if you like. And the big, and from my point of view, I mean, I don't know if I can, I think, anyway, I don't know if I'll ever be able to convince them or what they think of it these days. But the problem was that they were thinking of connecting this to physics purely from the Minkowski space-time side. So they're looking at Minkowski space-time twisters, Minkowski-space-time spinners. And those, the twister theory just didn't, if you just look at Minkowski-Space time, you don't see the sort of new things, which I'm finding interesting, which I think tell you something new about particle physics. You don't see this kind of internal, the fact that one of these factors can be an internal symmetry. You just can't see that in Rikowsky space time. And then so, and then there's some other more technical things about, I better not get into that. But the, there's kind of, well. It's okay. The audience is generally extremely educated in physics and math. Yeah, I would actually, well, maybe to connect this to what I'm saying, right, is I think, you know, also the way people think about general relativity in, you know, in Cassidy's signature, general relativity is not a chiral theory. It's supposed to be right invariant, parity symmetric theory. So the problem with thinking about general relativity in terms of twisters is that your setup is completely a chiral. So you can, you naturally end up with, if you try and do gravity with it, you end up with kind of something that's not quite the right theory of gravity. It's kind of a chiral version of gravity. Anyway, this is a very interesting story, but I think Penrose always referred to this as the Googly problem. Right, right. Something about cricket. Yeah, and cricket, there's something about how the balls. We're North American, so yeah. Yeah, so anyway, but so if you know about cricket, you can definitely, maybe this makes more sense to you, but he always referred to this as a Googly problem that he was kind of, in the twister theory, he's only getting one, he's only getting things spinning one way. And, but anyways, but you can see from my point of view, that's evident that was always, that's evidence of exactly what I'm trying to say now that, well, space time is right-handed. Yes. Yeah. So it's a related problem. But that was always kind of a, so Penrose and the people around them, I think, put a lot of effort into trying to revamp twister theory into something chirally symmetric. Now, why would they want to do that if the standard model isn't? Well, they weren't really trying to describe the standard model. They never really have to do. They thought Twisters were way of thinking about space time, so they wanted to do general relativity. And general relativity is not a chiral theory. So they were trying trying to find kind of a how do we get rid of all this chirality and uh and they never really successful at that so you're saying it's a pro not a con yeah yeah exactly it's a feature not a bug yeah right right but in terms one interesting fun thing that the sociology though is that what um know, so the idea that you could get, use twisters to quantify, to do general relativity and perhaps quantize it, that was always something which, you know, Penrose and his people were working on, but, you know, most physicists, I think, felt that wasn't really going anywhere. This wasn't going to work. And maybe Witten was probably one, was an example of somebody I think who really could see the mathematical power of these ideas and how important they were as new ideas about geometry. Again, that's a general statement that can be said. And then Ed Witten saw the power of this mathematics. Yeah. Yeah. Well, so he, I think even going back to a postdoc, he learned about Twisters, he was trying to do some things with it. But he never kind of, but he then actually finally found something. And this was about 20 years ago. And what became known as the Twister string. So he actually became, he found a way of kind of writing, you know, a different way of writing down the, um, perturative calculations in Yang Mills in terms of, um, of a sort of string theory, except it's a very different kind of string theory than the one that, the one that's supposed to be the theory of everything. And, and it's a theory where the string lives in twister space. So, Written wrote this really kind of beautiful, very, very beautiful paper about twister string theory. And so, and so since Witten is talking about twisters, of course, all of a sudden there's a lot of physicists who were never had, I think, good to say about twisters, or all of a sudden are rushing out to learn about twisters. So that, and there's, but there's been an ongoing story of, um, of this twister string story, which is a lot of people have done a lot of things. But again, a lot of it hasn't really worked out the way people like, and for the same reason as Penner, that Penner's always had, that the people are trying to quantify is a chirally version, a chirally symmetric version of general relativity using this thing. And that's not what it really wants to do. So anyway, but that's sociologically very important about why most high-energy physicists have more, have heard about twisters and don't, and often have nice things to say about them is because of the twister string. Okay, there are quite a few questions that I have. Okay, one is the particle physicists' repudiation of twister theory or just distancing from it because it's not useful to them. Is that something that they also slung at string theory or were they more embracing of it? Wait, so, I'm not quite sure. Who do you kind of mean? Who do we, are we talking about you? I'm not sure. Earlier, you said that the particle physicists weren't initially adopting string theory, sorry, twister theory because it didn't provide them with anything that's new. You said, well, okay, if we need to do some conformally invariant calculation, we'll use twister theory. Yeah. But at the same time, string theory is known, or at least colloquially known for not producing what's useful to high energy physicists, but useful outside of high energy like to mathematics or maybe condensed matter physics but what I'm asking is around the same time when they were distancing themselves from Twister theory or not using it were they then embracing of string theory or they gave the same critique well okay so we have to you should start if we're talking about string theory yeah that's a kind of complex, this is kind of a complex story. And it has the whole story of particle physics and string theory, that's pretty much completely disconnected from twisters because, I mean, the issues that, that, that, you know, people, about why people were doing string theory or why they might mind or might, I want to do string theory it really had nothing to do with twisters the twisters is kind of a yeah anyway especially a geometric framework and then you know and then twisters kind of make a small kind of appearance due to witten at one point 20 years ago but that's kind of about it um yeah so i mean i i i can maybe we we can start talking this about the whole string theory and particle physics business, but I'm not twister. Anyway, just twisters, it seems like a bad place to start. I'm not trying to mix up twisters with it. What I just meant to say was it's interesting what gets accepted and what doesn't. Yeah. And so why was string theory accepted? Take us through the history of that. And also you could tell people who may have just heard the name, sorry, Ed Witten. But all they know about him is that he's a genius. But they don't realize that influence that he has. Yeah, okay, so this is a good place to start. Yeah. And, you know, Witten is really kind of central to this story. And so, you know, I think the short summary of the history of this subject of particle physics was that, you know, by 1973, you had this thing called the standard model, which was this, you know, incredibly successful way of talking about particle physics and capturing everything that you see when you, you know, in these, when you do energy physics experiments. And the story And the story, when I kind of came in, it feels, I went to start learning about, probably started reading books and things about what's happening and particle physics probably right around the mid-70s. I went to college in 75, and I spent most of my college career, a lot of it learning about the standard model and this stuff and then and um so by but but by the time I left grad grad school set I mean by time I left college in 1979 and I went to graduate school at Princeton people were starting to get yeah people had had spent had now spent you know sit let's just six years let's say trying to figure out how to do better than the standard model and one one thing is how to do find some kind of new anyway how to do better the standard model as a theory of particle physics but also but one thing is the standard model doesn't give you a quantum theory of gravity so the other thing thing was, how do we get a quantum theory of gravity? So these were kind of the big problems that are already in the air. And Witten, you know, so Witten is a genius. And he had been a grad student in Princeton. He actually came to Harvard as a postdoc, I think, in 77, 78. And I met him when he was actually was a postdoc. And he quickly started doing some really amazing things. I went to Princeton 79. A year or two later, he actually, you know, he went directly from a postdoc at Harvard to becoming a full professor at Princeton, becoming a professor of Princeton very quickly. And so the years I was in Princeton as a graduate student were from 79 to 84, and those were years, you know, people I think were getting more and more frustrated. There are lots of ideas coming up, but every idea that people kind of tried to do better than the standard model, or maybe to quantize gravity, really didn't, you know, didn't quite work. I think there's a lot of, and people were kind of cycling every six months through. There's some new idea you'd work on it for six months or a year, and people start to realize, well, this doesn't really do what we want to do. Let's find something else. So there were a lot of new ideas, but nothing really working out. But Witten then, you know, he had been interested. There was this idea that was very unpopular. There were very few people were working on to try to quantize gravity and unify it with the particle physics through string theory. And so it was, you know, people like John Schwartz and Michael Green were working on this, but it was a very small group of people, and there wasn't much attention being paid to that. But, you know, Witten was paying attention to. I think one thing to say about him is that besides being very, very smart, he's also somebody who can, you know, read people's ideas or talk to them and absorb new ideas very, very quickly. So, you know, he was kind of also spending a lot of time looking around, trying to see, you know, what other ideas are either out there. And this was one that he got interested in. But for various reasons, technical reasons, he thought, you know, this, there's a technical reason, so-called anomaly calculations about why this is not going to work out. And what happened right in the fall of 84, I actually went as a postdoc to Stony Brook. And the right around that time, Green and Schwartz had done this calculation that showed that these anomalies canceled, except there's some specific case where these anomalies canceled. And so Witten then became very excited about the idea that, you know, you could use in that specific case of this so-called super string theory to, yeah. so so so so so so witten heard about this and he said it said okay you know the thing that had in my mind why super string theory couldn't work as a unified theory and now it looks maybe like maybe you can get around that so he kind of then started working full full time on trying to you know come up with models or understand super string models that you could use the de unification. And so throughout kind of, I was now at Stony Brook, but I was kind of hearing reports of what's going out at Princeton. And throughout late 84, 85, 86, this was, you know, Witten and the people around him, this is what they were working on, Bobora. And they were, you know, they had a very specific picture in mind. It was that, you know, the super string only is consistent in 10 dimensions. You, so you can get rid of four of them by the so-called Calabial compactification. And hopefully there's only a few of these collabiaws. And one of those is going to describe the real world and you know we're all going to have this wonderful beautiful unified theory using this kind of six-dimensional geometry of claudeaos and we're going to have it within the next year or two and that was the way they were thinking and you know a lot of the people you know friends and colleagues of mine who you know we're doing kind of the thing that you would often do is go down and go you know when you're in princeton go talk to witten and say here's here's what i'm working on you know can you what do you think about this and i got several of them reported back to me yeah you know i went down to prince i talked to whitton and he said well you know what you're working on that's all right i said well it's good but know, you really should be working on string theory because that's actually, you know, we're all the actions and that's really, and you know, we're almost going to have the theory of everything there and you kind of work on string theory. So, you know, this just had a huge effect. So, and this was called the so-called first super string revolution. And, you know, there's a story over the next five or ten years of how, you know, people were brought into this field and people, some people are always skeptical. But, you know, it kind of gained more and more influence and became institutionalized during kind of the decade after that. And in some sense, the weird thing, the weird thing, the weird thing that's hard to understand string theory is why, you know, once it became clear, these ideas really weren't working out, why didn't, you know, this just fall by the wayside and people go and do something else? But 40 years later, we're still, it's still here. And so it's a very strange story. So what do you see as the main physics, physical problem or even mathematical problem of string theory? Do you see it as, well, how do we search this landscape or how do we find the right manifold, the six-dimensional caler manifold? Yeah, I think that was always the thing that bothered me about it from the beginning, which I think is the fundamental problem. It's, and it's a fundamental problem whenever you decide to do to use higher dimensional Romanian geometry, if you, I mean, this actually goes back to Einstein, Einstein and these Kluza Klein models. You know, people have often said, okay, well, you know, we had this beautiful theory of fourdimensional geometry in Einstein general relativity, and we have this particle physics stuff going on, which seems to have some interesting geometry to it. So let's just, let's just add some dimensions and write down a theory in five or seven or ten or whatever dimensions, and then do geometry there, and that's going to solve, and that's going to be the unified theory. So I mean, this is sort of thing Einstein was thinking about. But if you start thinking about this, the problem is you realize that these kind of internal dimensions that the geometry of particle physics and the geometry of special relativity are quite different. They're not, you know, they're these metric degrees of freedom in four dimensions. And if you try and you don't really have those in, in like in the standard model, you just doesn't have things like that. So if you put those sort of dynamical variables into there, the ability for these for these other dimensions by the four one to two, you you have a vast you you hugely increase the number of degrees of freedom and you have a theory with where you have to now explain why all this extra geometry which you've put in there and and which you're only trying to get a kind of small kind of very rigid kind of couple pieces of information out why is are all these infinite number of degrees of freedom why how can you just ignore them how can you you have to find a dynamics consistent dynamics for them and then you and that consistent dynamics has to explain why you don't see them yeah and so that's always been the problem with like Kaluza Klein models and with any kind of extra dimensional models. And string theory just kind of has this problem in spades. You know, you're instead of feel, instead of point particles, you have strings. They have a huge number of new degrees of freedom. You have to say that, well, the string vibrations are all happening at such high energies we can't see them. And then the extra 60, then they're trying to use the fact that super strings have very special properties in 10 dimensions. And they're trying to use that to argue that our strings are moving in 10 dimensions and that four are the ones we see and six are going to be described particle physics and so anyways it becomes a very complicated theory you have to write down in order to kind of make any of this work and make any of this look like physics and the from the beginning there was kind of no story about why is anything that looks like the real world going to drop out of this, you know, and why that? And that's still the case 40 years later. And the whole thing just suffers from this problem that you don't, you don't actually have't actually have the theory there's kind of a when you say that you have a string theory and people say oh we have this mathematically elegant well-defined unique theory they're talking about that's not a full theory that that's that's a perturbative limit of a theory and so what they really need in order to answer the questions they want to answer is they need something more general, a so-called non-perturbitive kind of general version of string theory. And sometimes people call it M theory. So if you want, we can call it M theory. And they need an M-theory. And nobody knows what M-theory is. No one has come up. You can write down a list of properties that, you know, M, M theory is supposed to be some theory with this list of properties, but you can't actually write down a theory. And so on the one hand, you don't actually have a real theory that you can nail down and say, this is a theory, we're going to solve it and look at the solutions and see if they look like the real world. So what you, what people end up doing is saying, well, we don't really know what the theory is. Let's assume that, but it seems that maybe there's one that has some properties that look like the real world. So let's work with that. And then try to constrain, see what constraints we can get out of it will tell us, you know, are we seeing something like the real world? And then they just end up finding that, no, there aren't really useful constraints that you can get almost anything out of it. So you get this landscape of all possibilities. Yes. And then, you know, 20 years ago, things got very weird when people just started to say, well, you know, instead of saying that normally if you have a theory, it can't predict anything because, you know, almost everything is a solution to it. You say, okay, well, that was a bad idea and you move on. Instead, you saw people saying, oh, well, it just means the real world is, you know, almost everything is a solution to it. You say, okay, well, that was a bad idea and you move on. Instead, you saw people saying, oh, well, that's, it just means the real world is, you know, all of these possible things exist in the real world and the multiverse and, yeah, and just for, you know, for anthropic reasons, we happen to live in this random one. And, you know, I mean, anyway, it's, the fact that anyone ever took any of that seriously is just still kind of, I don't have any explanation for it. It's just far. Yeah. Okay, so to summarize, somewhere around, this is not a part of the story that was said, but somewhere around the 1960s, some amplitude called the Veneziano, I think, Veneziano. I don't know how to pronounce it. Yeah, the name. Venetia. That was the first inklings of string theory and it had to do, was come up with because of the strong force. They were trying to solve something. Then it was forgotten about. And then around the 1980s, there were some other problems with string theory that were solved. And so this is the Green Schwartz anomaly cancellation. Yeah. And then some people say that that was the first revolution. But it's also more accurate to say that that precipitated Ed Witten to take it seriously. And then that's what precipitated the first string revolution. Okay, then from there, then you realize that there are different ways, something like 5 to the 100 or 10 to the 500 or some extreme amount that if you were to do some naping calculation, all those books behind you, the amount of words ever written, not just books ever published, words ever written, I think easily letters ever written, like single letters, it would be like saying, find this one letter in every single book that's ever been written, including all the ones that have been on fire and underwater and so on. Okay, that's not such a problem if you can figure out how to reduce the search space. But if you can't, then it turns out the problem is NP-complete, which means you just have to brute force. Is that a correct summary? Well, actually, maybe you go back to one thing and say, yeah, so this is one part of the story I didn't say, is that string theory had originally come out as a potential theory of the strong interactions. And that actually was one reason Witten, I think, was looking at it, is that so one of the open problems that the standard model left open was how do you solve the strong we have this strong interaction theory but how do you solve it and it looked like maybe you could you could use the old ideas about strings to solve it and I actually spent a lot of time learning about strings as a graduate student because of that and I was really to Witten but but the with um this kind of multiplicity of solutions of string theory of is that it's not just that there are too many of them it's just that you don't actually have a definition of the problem you know so so people this kind of drives me crazy people often talk about well the problem is that we don't know how to put a measure on the space of solutions of string theory. And if we could put a measure, then we can figure out, you know, maybe it's concentrated someplace. Right. And that would be great. But I keep pointing out that the problem is not that you don't have a measure of the space. The problem is that you have no idea what the space is. As I was saying, you know, to even define what a string theory solution is requires knowing precisely what M theory is. You don't know it. There are no equations anyone can write down which you say, you know, if we were smart enough and we look and could find all the solutions to this, this would, you know, these are all the solutions of string theory. You just don't have that. So all of the things that you do have, like you can go out and say, well, well, maybe it's these gadgets and you have 10 and the 500 of them or whatever. Those are all just kind of cooked together possible approximations to what you think might be a string theory solution. Those are not, there are solutions to some equations you've written down, which are not, they are not the equations of string theory. There's something you wrote down and think maybe these things have something to do with string theory. So the problem is much worse than any of these practical problems of there's too many of these things. And this whole business, and now it's become kind of an industry that, well, let's apply machine learning techniques to this. And it's just, I mean, you're just applying. Anyway, you're just. Does this frustrate you? Yes. I mean, this data is garbage. You know, so you basically are throwing, you basically do not actually know what your problem is. So you're cooking up something which you can feed to a computer, but it actually is kind of known to be garbage. And you're doing processing on this and producing more garbage and, you know, getting grants to do this and going around telling people that you're looking for the universe the universe I mean it's real that's just utter nonsense I'm sorry many people don't know because they don't know the history but since 2010s it's become somewhat cool to dunk on string theory at least in the popular press okay maybe not inside academia but you were alone you and Lee Smollin were lone wolves. Early lone wolves. Can you talk about that and talk about some of the flak you took, maybe still take? Yeah. Anyway, it was certainly a very strange experience, a very strange time. But, you know, I think the thing to say is that throughout, you know, I was never, I was always fairly skeptical about string theory, but, you know, initially for many years my attitude was, well, you know, throughout, you know, I was never, I was always fairly skeptical about string theory, but, you know, initially for many years, my attitude was, well, you know, who knows, you know, is certainly very smart. These people are, you know, they're going to, sooner or they'll figure out for them, either they'll figure out this works or they'll, or they'll do something else. But then, you know, just as time went by, years went by and that this was just not happening. And you had more and more and more kind of popular books you know i have to confess maybe in some sense it's somewhat of a reaction to uh to brian green who is a my friend and colleague here in at columbia but uh you know so he did a very very good job of with pbs specials convincing the world that you know this this was a successful this was an idea on the way to success when it really wasn't. So I thought, okay, well, somebody should, you know, sit down and write a book about, you know, what the real situation here is. And, you know, it's not like when I talk to people privately about this, you know, I would say that people who are not string theorists mostly would would say, yeah, you know, yeah, you're probably right. This is not, this doesn't seem to be going anywhere, but you know, whatever. And then the, and people, and when I talk to string theorists, I have plenty of strength theorists, they would often say, yeah, you know, yeah, there are a lot of huge problems and we just, we don't really know anything better to do right now is where we're going to keep doing this. But yeah, yeah, all these problems you're pointing out are really, yeah, they're real. And, um, so what's wrong with that? Well, it was, the weird thing was, I think was this disjunction, this disjunction between the private opinions of people, what people were saying to each other privately, what you, private had said, and what you were seeing in, in the popular press. And, you know, you've, so, and there was, you know, and one aspect of this was people not wanting to kind of publicly criticize something. And partly, and I think the subject became more and more kind of ideological. And the string theorists kind of started to feel kind of in battle. They were very well aware that a lot of their colleagues thought what they were doing was not working. On the other hand, you know, so they became more defensive. And there was a lot more it became. And a lot of people, I think, would tell me, yeah, you know, I agree with a lot of you're saying, but yeah, but don't quote me on this publicly. I don't want to get involved in, you know, in that mess and alienating a lot of my colleagues and who are, anyway, so, but I have this weird status that I'm actually in a math department, not a physics department, and, you know, I don't have a lot of the same reasons that you don't want to annoy some powerful people in physics, like, you know, trying to get grants, get your students jobs, et cetera, et cetera. It didn't really apply to me. So I thought, well, you know, if somebody is going to kind of start a lot of time thinking about this stuff. And I, you know, I spent a lot of time thinking about this stuff. And I started writing this in around 2002, 2003. And the book was finally published. It was a long story about getting it published, but it finally got published in 2006. And in the meantime, Lee Smolin had been writing a book. He was coming from a different direction. Trouble with physics? Yeah, the trouble with physics. And he had his own motivation, so it was trying to write something, I think, more general and sociological, but with this as an example, and I think the way he describes that the example kind of took over the general theory. And so he ended up also writing a book about string theory. And the books ended up coming out at the same time, which I think, you know, it was kind of a force multiplier there that, you know, people, if one person is writing a book which says, well, you know, a lot of the things you're hearing, you're hearing are not right. Or people say, well, that's just one person's opinion. But if two people are doing our same thing, everybody's like, oh, you know, there must be something to this. And so I think the combination of the two books, I think it did have a lot of effect on, it did make a lot of people realize there was a problem here. It made a lot of the strength theories, you know, much more defensive. I mean, it also caused, I think, a lot of people, young people thinking of doing string theory or people doing string theory to decide to move on to something else. But so people very often tell me that, you know, about effects this book had on them or other people they knew in terms of their decisions about what to do with their research or their career. The book is called Not Even Wrong. The links to all resources mentioned will be in the description, including this book. So you mentioned that your colleagues would talk to you privately, and then they would say something else to the popular press. Now, when you say popular press, are you also including grant agencies with that, like just the public in general? Because it's not just a popular science issue, it's also a grant issue where the money goes. Yeah, so it's not just the popular press. And to be clear, I should say, it's not that they would say one thing, one place, it's just, they would carefully just not say, you know, that there were things that they would say in conversation with me or I think in conversations with other people, not just me, that they would just say, okay, this is not something that. Okay, sin of commission versus omission. Yeah, it's not like they were going out and saying, oh, the strength theory is going great. It's just that, you know, anyway, they were, they were not kind of, they were not saying this is really appears to be a failure. But yeah, but you're right. This issue kind of occurs at all levels from, you know, the very, very popular press, from kind of television specials um to you know more more serious popular press or what what gets into scientific american you know what what gets into uh now we have quantum magazine you know which are more more serious uh parts of the parts of the press aimed at the more at the public to um you know all the way down to it yet to exactly um yeah like in grand proposal you know what what do you write in in grant proposals whatever or if you um if you're anyway you're trying to explain to some some kind of funding person or something about what you know what's what's going on in your subject do you um yeah and you know what do you say about string theory and so the you know the string theories i think have often you know that they've i think everybody whatever you're working on you're often forced by this business of getting your students a job or getting a grant to be you know know, to say, to go right up to the boundary of what's defensible and being optimistic about what you're doing. But, and there, you know, so that's what string theorists have certainly always been doing. You could argue, you know, in many cases, it's not different than what other scientists do. But it's, I think the thing which i i have to say i have found more and more disturbing the reaction of and this started when my book came out and i think lees small had a similar reaction the um i think both of us were expecting a much more serious intellectual response to the issues we were raising. We were raising serious, serious technical questions, and we were getting kind of back, you know, kind of, you know, personal attacks. From people in the community or from the public? From people in the community. I mean, I think, you know, what you're getting from people people who don't in the public don't know much about this you're you're getting some completely random combination of people who are annoyed because you're saying something different than what they heard and other people who become your fan because you're saying something different and so you end up sure sure you end up with a huge number of fans who you don't necessarily want as your fans. But anyway, yeah, so both of us were expecting, you know, that, you know, we put a lot of effort into making a, you know, a serious intellectual case about what these problems were. And instead of getting a serious response, we were getting, you know, these kind of personal attacks of how dare you say this. And so, for instance, there's one prominent blogger who decides who would write these endless blog entries about what's wrong with Peter White and what he's doing. And at some point, I was trying to respond to these. And at some point, I realized, you know, what this guy's talking about is nothing to do with what I actually wrote in my book. And then he actually kind of publicly admitted that he was refusing to, he refuses to read the book. So this is a, anyway, this kind of blew my mind. How can you be an academic and engaged in, you know, academic discussion, intellectual issues? And you're spending all this time arguing about a book and you're refusing to read it. mean how it's just really crazy and that was a string theorist yeah or just a colleague yeah okay string theorist yeah speaking of brian green oh sorry continue yeah yeah no yeah no yeah i didn't mean to suggest that no no no but but anyway that's just one example so and i and i think, you know, this is an ongoing, I think, disturbing situation that people are just not, people are kind of defending that field and continued and research there with just kind of refusing to acknowledge the problems or to have kind of serious discussions of it. I think, you know, on your last year, your last thing with Edward Frankel, I think, it's kind of funny because he, you know, I know him and I actually was out visiting him in Berkeley in June or something and were talking about things. And he told me, oh, Peter, I'm, you know, I'm going to go to the strings conference. And it's the first time I've been to a strings conference and now. And, you know, he's heard me go on about this for, and he's kind of nodded his head politely. And, you know, he's saying, well, I'm a mathematician. I'd rather than not, you know, but this sounds a little bit of it. Maybe he's published with Witten. Yeah. And then, you know, so he, and he knows all these people. And he knows a lot about the story, but he, and I think, you know, he knows me well know enough that I'm, you know, I have a somewhat, I'm not a complete fool and I have a somewhat serious point of view, but, you know, maybe I'm really a bit too extreme about this. But then he went to the, this conference. Then when it comes back, he gives me a call, it says, basically, you know, Peter, I didn't realize how bad it really was. You're right. This really is as bad as you've been saying. So it was, anyway. What was bad? The exuberance of the young people or the old people telling, misleading the younger people into a useless pit? Or like, what was, what was bad? Yes, it is as bad as you say. Well, I think what's bad is really just this kind of, this kind of refusal to admit, I mean, this is a field which inflexia has serious problems. Things have not worked out. These ideas really have failed to work. And instead of admitting that, ideas have failed and moving on, people will just kind of keep acting as if that's not true. And so the, you know, I think... Sorry to interrupt. I'm so sorry so why would edward expect an admittance of the failure of string theory at a strings conference i think one thing to say you know i mean part of the story about him is you know he's a mathematician and and you know so mathematicians if you do mathematics the one thing you have to be completely clear about is you know what you understand and what you don't understand and what is a wrong idea and what is a right idea. You know, and if something doesn't work and is wrong, you have to, you can't play a game. You cannot play any games about this. This is, you know, you have to admit that this is wrong. And so I think especially for mathematicians to come in and see an environment where there's You know the kind of guiding ideas that people haven't really haven't really worked out and a lot of things you know are known do not work for known reasons but people are still kind of acting as if this is not true and trying to figure out how to kind of do something and make career for themselves in this environment it's a very no i think he he he recognize that but it is part of it is the um i mean mathematics is a very unusual subject that people things things really are wrong or right and you and you're you know it's you absolutely absolutely cannot seriously make progress in the subject unless you recognize that and uh and mathematicians are also much more used to um they're much more used to being wrong i think one of my colleagues john morgan likes to say that uh you know mathematics is the is the only subject he knows of where you know if two people disagree about something and they each think the other is wrong, they'll go into a room and sit down and talk about it, and then they'll emerge from the room with one of them having admitted he was wrong, the other one was right, and that this is just not, it's not a normal human behavior, but it's something that is part of the mathematical culture. Earlier I said, speaking of Brian Green, and what I meant was I had a conversation with Brian Green about almost a year ago now, and I mentioned, yeah, so Peter White has a potential toe, Euclidean Twister unification, and then he said, oh, does he? Oh, I didn't know. He is in your university, not to put you on the spot, but why is that? Well, it said aloud, I don't think it's true by the professor of physics, mainly who studies string theory. Well, there are so many proposals for toes. Yeah, there are proposals in your inbox, but there aren't serious proposals by other professors. There aren't that many serious proposals of theories of everything, at least not on a monthly basis. Well, I mean, I mean, this is this really doesn't anything in particular to do with Brian. You could ask, you know, since, you know, people in this subject, you know, in principle should be interested in this. There's, I've gotten very little reaction from, from physicists to this. And, and in some sense, it's kind of clear, clear why. I mean, it's kind of clear why. I mean, I wrote this paper. I've read it by the blog. And, you know, I've gotten no reaction. In both cases, I don't have reaction from people telling me that I've talked to about or saying, oh, you is this this is wrong this can't work for this reason but well i think that this is this is very very much the problem with the the paper that i wrote about this it's very it uses some quite tricky understanding of how twisters work and twister geometry works, which is not, is something a very few physicists have. So Brian, it would, I'd be, I'd be completely shocked if Brian actually really understood some of the things going on with twisters that I've been there talking about. And the problem, I think for anybody who then, if somebody comes to you and says, oh, I have this great idea, it involves, you know, these subtleties of twister theory. And you're like, well, you know, I'm really not in the mood to spend a week or so sitting down trying to understand that subtle is a twister theory. So I think, you know, maybe I'll just nod my head politely and go on my way. That's part of it. And then part of it is also that a lot of, you know, this is very much a speculative work in progress. I'm seeing a lot of very interesting things happening here, but I'm not, in no sense, have completely understood what's going on or have the kind of, you know, understanding of this where you can write this down and people really understand, can follow exactly what's going on. So it's not too surprising i haven't got that much i can see why understand the typical reaction to this and um brian is someone of a special case because i mean he also actually is very um i think actually he actually a lot of his effort is as has in recent years has gone into other things especially the the World Science Foundation Festival, I think, is now more or less, you know, it's kind of most, it's mostly Brian Green at this point. Yeah. And then it's, so he's, anyway, he's thinking about other things. And I have very, I don't have very little contact with people in the physics department. I mean, they're mostly thinking about very different things. And it's kind of a sad fact here at Columbia, but it's true essentially everywhere else that the, you know, the mathematicians and physicists really don't talk to each other. They're really separate silos, separate languages, separate cultures, and places where you have kind of mathematicians and physicists and kind of active and high-level interaction with each other is very unusual. It doesn't happen very much. I have a couple questions again. I'll say two of them, just so I don't forget them, and then we can take them in whichever order you like. So one of the questions is how slash why did you get placed into the math department? So that's one question. And then another one is, you mentioned earlier that Witten has this power to survey a vast number of people and extract the ideas at great speed. And so a large part of that is raw IQ, like sheer intellect. But is there something else that he employs like a technique that you think others can emulate? I imagine if Witten was to read your paper, he would understand it. And I imagine that he would see, oh, he would see the benefit of it and maybe the application to string theory or maybe it offshoots in its own direction. But anyhow, so those are two separate questions. One about Witten, and then one about you and the department you're in. Okay, yeah, I've got, yeah, there are two very different. Let me start, let me just say something quickly about Witton, just saying about having dealt with him over the years. One thing that I find very interesting about him is just, you know, he travels around a lot. And, you know, he, let's just say, let's just say his way of socializing is to, you know, if he's come to a department and he's at T or whatever, he'll, you know, and he's introduced to anybody, he almost immediately will ask, okay, well, what are you working on? You know, explain it to me. And so just a lot of what, anyway, that's a lot of what he's done over the years has just been, has just been, you know, trying to really be aware. And, you know, I've said what I've done doing and tried to get him interested. He's, I know, he's, anyway, we'll see where that goes. Maybe I'll have more success with it with this new paper, maybe not. But he's, he's responded, though, or no? He has responded, but it's more that he's kind of looked at it. He actually, the first version, he actually made some kind of comments more about the beginning of it. But I think he didn't engage with most of what I started talking about. We're going to get back to the math question soon, the math department question. But do you think a part of that is because there's a sour taste given your book? Yeah, yeah. I mean, I'm not, I mean, again, I've known him since I was an undergraduate. You know, I think, you know, he's, I think he's aware, you know, that this guy is not an idiot, but he's also, I'm also not his favorite person in terms of kind of, you know, the impact I've had on his, on his subject. And yeah, and I think, you know, he also, I think he understands it's not personal, but, you know, it's not, it's very hard to deal with somebody who's kind of, you know, been this kind of main figure, kind of telling the world that the thing that you think is your main accomplishment in life is wrong. So this is not, yeah, anyway, I'm not his favorite guy, but, but anyway, I can know, we're still. Sure. It's fine. He's, yeah, you know, I think he's a very, you know, anyway, he's a very ethical and very, and I think when I complain a lot of, a lot of, most of the worst of what the kind of, this kind of pushing of string theory in ways which, which really were completely indefensible. It's, he's mostly been not, you know, he's rarely been the worst offender in that. I mean, that's really more other people than him. But, yeah, he's a true believer. He's really enthusiastic about him. He still is. Okay. So to get back to my own personal story, so what happened, you know, so I got a postdoc at the Stony Brook Institute for Theoretical Physics in 84. I was there for four years, and that was the physics institute, but the physics institute was right above, it's the same building as the math building. And so, and the things I was interested in, I was trying to stay away from string theory and I was interested in some other things. And, you know, I was often talking, and I was trying to learn a lot of mathematics. I was trying to learn more mathematics to see if I could make any progress on these other problems. So I spent a lot of time talking to the mathematicians in Stony Brook. And some of them, you know, there are some really great geometers. There are some really great mathematicians. And I learned a lot from them. And it was a, that was a great experience. But at the end of four years there, you know, I needed another job. I did set out some applications for postdocs in physics, but the, I would say that that was kind of the height of the excitement over string theory. And especially somebody like me saying, you know, I'm really interested in doing something about the mathematics and physics, about applying mathematics physics, but I don't want to do string theory. That was just, that was not going I was not going to get any, any kind of reasonable kind of job that way. That's just not going to happen. So, anyway, so I ended up realizing, well, maybe the better thing, I'll have better luck in a math, in a math department, and I'm getting, and so I ended up going up, spending a year in Cambridge as kind of an unpaid visitor at Harvard partly, and I was also teaching calculus at Tufts. And so then I had some kind of credential, okay, well, at least this guy can teach calculus. And so I applied for a one-year postdoc at the Math Institute in Berkeley, MSRI, and I got that. And so I spent a year. Is that how you got to know Edward? No, no, he wasn't, that was before him. I mean, he would have still been at Harvard and a much more junior person. Yeah, yeah. Yeah, he came to Berkeley later. Yeah, no, that was like 80, 88, 89. But that was an amazing, that was actually a fascinating year because that was the year that Witten had come out, Witten had kind of dropped string theory for a while and was doing this topological quantum field theory stuff in Turing Simon's theory. And he was doing the stuff which won in the Fields Medal. And, you know, it was just, just mind-blowing, bringing together of ideas about mathematics and quantum field theory. And so most of the year was devoted to learning about that and thinking about that. And, you know, Witten came and visited and Atiyah was there. And actually a lot of chance to talk to him, which was wonderful. And so that was a really fascinating year at MSR-I. And partly because so much of this was going on, you know, math departments were more interested in hiring somebody like me, even though I didn't have the usual credentials because they felt this is somebody who actually understands this new subject, which is having a lot of impact on our field. So Columbia hired me to this non-tenure track for your position. And so I was to do that as I was teaching here. And after a few years, again, I was getting the point, okay, well, now I've got to find another job. And they, so the department needed somebody to, they'd set up a position for somebody to teach a course and maintain the computer system. And I said, well, you know, I can probably do that. And that's not a bad job. And so I ended up agreeing to take on that position. And that's always been kind of a renewable position. It's not tenured, but it's essentially permanent renewable. And I've gone through various kinds of versions of that since I've been since the 90s and it's worked out very well for me I'm actually quite quite happy with how it's work but it's a very unusual career path and it it has given me a lot of insulation from the normal kind of pressures to perform in certain ways and to do certain things allowed me to get away with all sorts of things, if you like. Like what? Well, like writing a book called Not Even Wrong, explaining what's wrong? How did that come about? So, for instance, this is going to be incorrect because I'm just making this up, but then correct it. For instance, you're walking along someday. You have this idea. Maybe it's a splinter in your thumb for a different reason about string theory. So then you go to a publisher and you say it or you say it to a journalist and then the journalist hears it and they say you should write a book and you say maybe, then you think about it, you start writing a chapter. The nitty-gritty details, how does that happen? How did it go from Peter White, mathematics professor, to then writing this popular book? Well, so yeah, let's say throughout the 90s, you know, I was very much, you know, I'd always, you know, I was interested in the same kind of question as can you do different things in math and physics? I was trying to follow what's going on in physics. I've been trying to follow what's going on in string theory. And I was getting more and more frustrated throughout the late 90s at this, what I would see in the public and what I would see, or just to not reflect my own understanding of what actually was going on. And partly I kind of mentioned, you know, so there's a, for instance, Brian's PBS special about the earlier's. I mean, it just, that just just seemed to me to be giving that just didn't really didn't agree at all with what i would actually saw going on and so i thought well somebody you know somebody should write this up and i would have hoped it would be somebody else but then as you go along with no one else is going to do this and you know i'm actually pretty well placed to do it for for very reasons and started thinking about it and i think around 2001 i actually wrote kind of a short thing that's on the archive of kind of you know a little bit of a kind of polemical several page thing you say look here here's the opposite side of this right here's what's this is really not working and here's why and that that was the beginning of it and like i got a lot of reaction reaction to that. And I started to more and more feel that the right way to do this was to actually, you needed to write something kind of at book, sit down and at book length explain exactly what's going on. And I also wanted to do something also more positive to try to explain some of the things that I was seeing about how mathematics, you know, there were some very positive things happening in the relationship between mathematics and physics, which has some connections to string theory, but were also quite independent, like Wittance-Turne-Syman's theory, for instance. So I also wanted to also write about the story of what's going on in this kind of physics and this kind of fundamental physics, but kind of informed by, you know, someone who's actually spent a lot of time in the math community and informed by a lot more mathematics than as usual in this thing. So there was kind of a positive. It's rarely noticed, but there are a bunch of chapters in this book like on top logical quantum field theory, nothing to do with string theory, which nobody really paid much attention to or understands. But anyway, so I wrote this, and I was, so I just said, well, I'll just write this thing. And I think I, around then, I may have also had a friend who, he'd done a book proposal and written a book. But by the time he actually was writing the thing, you know, he was just kind of sick of it and he didn't really want to be writing it, but somebody had given him in advance and he had to write the book. So I thought, well, I don't want to do that. I'm not going to go out and make a proposal to a publisher. I'm just going to write when I want to write. And we'll see how it turns out. And I think know, I think we'll see if someone wants to publish it great. And so then I was getting to the end of this and somebody from Cambridge University Press showed up. He was just in my office going around asking people, you know, what are you working on? Is there some kind of book project we could work on? And I told him about what I was doing. And he got very, very interested in it. And so it actually then became, you know, Cambridge University Press was then considering it for a while and they sent it out to various reviews and the reviews were kind of fascinating. There were half the reviews said, this is great, this is wonderful. Somebody is finally saying this is fantastic. And the other half said, oh, this is absolutely awful. This will destroy the reputation of Cambridge University of Press. Interesting. And the problem with the University of Press is, you know, they're not, they're actually not really, they're not really equipped to deal with that kind of controversy. I mean, they, they've got, they have like boards of so-and-so that have to vote on everything and they're very pretty conservative institutions. So at some point it became pretty clear that things were not going well there. And so I sent it around to a bunch of people. And anyway, and one person I sent it around to was Roger Penrose. And he ended up getting interested in it and asked me if he could send it to his publisher, and they ended up publishing it. Oh, great. Yeah, he's not a fan of string theory either. No, no. Yeah, so he definitely agreed with me about that. Yeah. Now that you're in the math department, is that what allowed you to see the connections between Twister Theory and the Langlands program, or is that something that existed before? Oh, well, I mean, the connection, not the Langlands program. Obviously, that goes back to Langlands. Well, oh, no, whether there is, I think it's still, you know, whether there is any connection between Twister Theory and the Langeland program, that's a very, that's extremely speculative idea and fairly recent one, I would say, yeah. Yeah, so that. What aspect of the Langlands program? Like the local or geometric? Maybe to back up a little bit. I mean, so the Langlands program is, anyway, this amazing story, I guess you heard a lot about it from Edward, but it, it's one reason I got into it is it became more and more clear to me that the right way to think about quantum mechanics and quantum field theory was in this language of representation theory, that that was the language of, and then it started to, okay, well, I should learn as much as possible about what mathematicians know about representation theory. And, and you, you, you, you, you, you find out about the language program, and the language program is saying that all of the basic structure of how the integers work and how numbers work and things is, you know, closely related to this representation theory of lead groups and in this amazing, amazing way. And there's just an amazing set of ideas that ideas behind the Geometric Langlands program, which, you know, they have a lot of similar flavor to the things I was seeing in some of physics. So it was, you know, I said, it's just been a many, many years process of slowly learning more and more about that. But that stuff never really had anything to do with twisters. And so the one, the interesting, the interesting relation to twisters is that, you know, I had actually, I'd actually written this paper and I'd given some talks about, um, about the twister stuff. And I pointed out that I'd pointed out that in this way of thinking about things, there's a thing that I told you that a point, a spacetime point, is supposed to be a complex plane. Well, if you take this, actually in Euclidean space, it's something you can think about it a complex plane or you can mod out by the constants and use the real structure of Euclidean space. And you get something, a geometrical object corresponding to each point, which is called a twister P1. It's basically a sphere, but you identify opposite end points of the sphere. And so I'd written about that in my paper and some of the talks I was given, I kind of emphasize that. And then so then I get an email one day from Peter Schulza, who's one of the people who's making this really great progress in the language program in number number theory and it's been coming up with some of these fantastic new ideas relating geometric langlands and arithmetic langlands and he said and he basically said yeah I was looking at this talk you gave and you know it's really nice about this geometry and seeing this Twister P1 going there said what's amazing is this Twister P1 is exactly that same thing as showing up in my own work. You know, if you, there's this work he was doing on the, on the, on the, on the gym, the geometric langlands. And if you specialize to what happens kind of as a, at the infinite prime or at the, the real place, not, not at finite primes, the structure he was seeing was exactly the twister P1. So, I mean, he kind of pointed this out to me and asked me some other questions about this. I don't think I could tell them anything useful, but that kind of, that did kind of blow my mind that, wait a minute, this thing that I'm looking at in physics, that exactly the same structure is showing up in this, in this really these new ideas about geometry of numbers. And so I then spent a few months kind of learning everything I could about that mathematics and the Twester P1, and I'm still following it. But, you know, I should say that, you know, to my mind, it's just a completely fascinating thing that these new things that we're learning about the geometry of number theory and these speculative ideas about physics that you're seeing a same fundamental structure on both sides. But I have no, I mean, I have no understanding of how these are related. I don't think anyone else does either. Yeah. Have you asked Peter if he would like to collaborate? Well, there's not. Is that like uncouth? No, but I think he and I just have very, you know, I mean, too incompatible? No, no, no. It's just, you know, he's doing, you know, he's doing what he's doing. I mean, I mean, first of all, I mean, one thing to say is, you know, he's having such incredible success and doing such amazing stuff that, you know, interfering in it with that anyway and telling him about, oh, why don't you stop doing what you're doing and do something? And I'm interested in. It seems to be a really bad idea. Anyway, so he's doing extremely well doing what he's doing, and most of what he's doing isn't related to this. He really, really understands in an amazing way what's going on with the geometry of peatic numbers and these things like this, which I don't understand at all. And he's just been revolutionizing that subject. And it's something I can only kind of marvel at from the distance. The kinds of issues that were on kind of stuck that are kind of for me are actually much more, they really have nothing to do with his expertise. They're really kind of more, more, you know, I probably should be talking to more physicists or whatever. So he's, yeah. But I mean, it's certainly, I think it's in the back of his mind, oh, you know, this stuff that I'm seeing, I should every soften look and think about what if I can understand the relation to physics. And it's in the back of my mind, the stuff that I'm seeing physics, I should try to keep learning about that number three stuff and see if I see anything. But that's really all it is. But a lot of this is very new. I just heard from him a few weeks ago that, you know, he actually, he actually has some new idea about this particular problem from his point of view. And he was supposed to give a talk about it on last Thursday at this conference in Germany. And I'm hoping to get a report back of that. But this is all very active and very poorly understood stuff, but it's not, but definitely the connection between math and physics here is very, very unclear. But I'm, if there is one, it will be mind-blowing, and I'm, I'm, it's certainly kind of on my agenda in the future to try to learn more and look for such a thing. But I don't have anything positive to say about that, really. So I want to get to space time is not doomed. There's quite a few subjects I still have to get to. I want to be mindful of your time. But how about we talk about space time not being doomed? It's something that's said now. I don't know if you know, but there's someone named Donald Hoffman who frequently cites this. He's not a physicist, but he cites it as evidence or as support for his consciousness as fundamental view. And then there's Nima Arkani Ahmed, who's the popularizer of that term, though not the inventor. Yeah. So maybe to, I mean, I can kind of summarize that. Yeah, so I don't really have anything useful to say about, but Hoffman. I mean, so he's interested in consciousness and other things I don't really have too much, I don't really know much about or I'm useful to say, but maybe to say what the, I mean, this has become, and I mean, the reason I wrote that there's this article you're referring to about space time is not due. I wrote partly because I was getting frustrated at how this had become such a, such kind of an ideology among people, among people and working in physics on quantum gravity, this idea that, and I think one way I would say what's happened is that. So when people first started thinking about how do you get quantized gravity, how do you quantum gravity? So the initial, one of the initial ideas was, well, you know, we've learned that we have this incredible successful, successful standard model. So let's just use the same methods that work for the standard model and apply them to gravity and we'll do that. And so it's going to be, anyway, and you're thinking of space and time in this usual way. And then there are these degrees of freedom that live in space and time, which tell you about the metric and the geometry of space and time. And you're trying to write a quantum theory of those things living in space and time and i think you know anyway people tried to do this there's lots of problems with doing it it's an incredibly long story string theory was partly reaction to the story but even string theory was still a theory of strings moving around in space and time so you weren't yeah i, you were still starting thinking in terms of a space and time. But more recently, you know, as string theory hasn't really worked out the way people expected, there has been this ideology of, oh, well, let's, you know, let's just get rid of this space and time somehow. And then we will write some theory in in some completely different kind and in the low energy limit will recover space and time as some kind of effective structure which you only see at low energies and that's become almost an ideology like our Connie Howlett likes to say space time is doomed you know meaning the the truly fun well theory is going to be in some other variables and space-time variables. He has his own proposals for this about these geometrical structures he's using to study amplitudes. But I don't, anyway, the things that I'm doing, you actually do get a theory. It looks like gravity should fit into this, and it will fit into this in a fairly standard way. This is standard space and time except, you know, the twister geometry point of view on it and interesting things happening with the spinners you didn't expect, but it's still, there is a usual idea that's about space and time are there. So my general feeling with the, the problem with this whole kind of space time is doom thing is you have to, you have to have a plausible proposal for what you're going to replace it with. It's all well and good to say that there's some completely different theory out there and the theory people used to is just an effective approximation. But, you know, first you've got to convince me that your alternative proposal is it works. And the problem is that people are just doing this without any kind of, you know, without any kind of plausible or interesting proposal for what it is you're going to replace space time with. And often, and often it even comes down to this crazy level of kind of this multiverse thing. I mean, you know, we have this theory where everything happens, so fundamentally everything happens, but then effectively you only see space and time. It's kind of, you know, you can say words like that, but it's kind of meaningless. Why is it that they have to come up with a decent proposal or replacement? Why can't they just say, look, there are some, with our current two theories, there's an incompatibility that suggests that spacetime, quote unquote, breaks down at the plank level or maybe before. So, for instance, NEMA's argument that if you were to measure anything with classically, you have to put an infinite amount of information somewhere, and then that creates a black hole. And then there's also something with the black hole entropy that suggests holography, but that doesn't mean space time is doomed. It's just a different space time. Yeah. Yeah, but for my point of you, I mean, what has been come to focus of that field a lot is this is are actually quite tricky, you know, very non-perturbitative, very kind of strong field problems about, you know, how, you know, what's going to happen to the theory when you've got black holes and black holes are you can. And so you've kind of moved away from, I mean, but, but the problem with the inconsistency between quantum mechanics and generalativity is a different, that is normally the one everybody worries about is normally a different problem. It's a very, very local problem. It's just that if you think of this in terms of the standard kind of variables, like what's the metric variables, and you use the Einstein the Einstein Hilbert action for the dynamics for these things if you try and apply standard ideas of quantum field theory locally to that at short distances you get these normal normalization problems and the theory becomes unpredictable so that's always been considered problem, how do you deal, how do you deal with that? But instead of having a proposal to deal with that and having a real kind of a new idea about what's really going to happen, what are the right variables at these short distances that will not have this problem? What are you going to do? They kind of ignore that, decided to ignore that problem and say, well, maybe string theory solves that problem. Who knows? And then to move on and to try to do something much, much harder, which is to resolve these issues about what happens in black hole backgrounds and stuff. And I don't yeah i i know but it seems to me a kind of a separate a separate issue you can still have space time and have these these these issues about you know what's going to happen in black hole backgrounds and stuff and you could still resolve them in different ways but but they're just, they really, it's a very frustrating subject, I think, to actually try to learn about it. You see people making these statements, and then you say, okay, well, what exactly do they mean? I mean, it's all well and good to say these very vague things about, this is doomed, and what about infinite amount of information, blah, blah, blah. But, you know, write down, tell me what we're talking about here. And there really isn't, it's almost a comically impossible to kind of pin people down on what is the, what are you talking, what theory are you talking about? And then finally when you pin them down, you find out that what they're actually talking about is they've, they're talking about some very, very toy model. They're saying, well, we don't know what's going on in four dimensions, so let's try it in three dimensions, and maybe two dimensions, maybe one dimension. And so they're talking about some comically trivial toy model, which they kind of ended up studying because, well, you could study it, and maybe there's some analogous problem happening there. And all they have are these kind of toy models, which actually don't seem to have any of the actual real physics of four-dimensional general activity in them. And that's what they're all studying these days. I see. Even Nima. Well, he's somewhat different, because he's coming at it from a different point of view. He's coming at it from this point of view of really trying to see find new structures in the in the perturbative expansions for, you know, for standard quantum field theories. So he's got a, he's got kind of a specific program looking at, yeah, I mean, he's not, he's generalized, he's not studying toy models. He's studying real four-dimensional physical models. But they're not, but, but, but they're often, they're generally models like Yang Milcery where you know exactly where the theory is. And it's not, this isn't solving the problem of quantum gravity or anything. It's well in theory. But I think maybe, I'm saying this a bit too quickly without thinking, but just to try to give a flavor of what I think he thinks he's doing, he's trying to take a theory that you do understand well, like Yang Mill's theory, and look at its perturbation series, Feynman diagrams, find new structures there and a new language, and then see if you can rebuild the theory in terms of these new structures. And then if you've got kind of a new way of thinking about quantum field theory in terms of these new different structures like his amplitude hydrant or whatever, then maybe you can then apply it. Once you've got a way of thinking in terms of those new structures, you can go back to the problem of quantum gravity and resolve that. Yeah. So I think, but, you know, I don't think he's not in any way as far as I know claiming to have actually gotten anywhere near there, but he's, and this gives you a lot to do. There's a lot of interesting structure, though. There's a lot to work on. And so he and his collaborators have done a huge amount kind of calculation with these things. But I, at least to my mind, I don't see them kind of coming up with what I think they hope to come up with, which is a different geometric language that really works and is really powerful that's going to get you something new. Did you listen or watch Sean Carroll's podcast on the crisis in physics? Well, no, I skimmed through the transcript of it. I was kind of wanted to see what he was. I mean, this is certainly something I'm very interested in. But, yeah, I thought, anyway, I thought the whole thing was actually quite strange because it's like four, four and a half hours long. And it's just him talking. So's just anyway I thought the whole thing that was actually very odd and it's something to do with kind of a the odd nature of the response to the um you know to to criticisms in the subject and so I think it was another kind of weird example it's you know there, he's kind of wants to say something about this issue of, you know, that many people are now, are now kind of very aware there is some kind of problem here and they're referring to it in the crisis and physics. But, you know, instead of, but, but, but just kind of talking about it for four hours or four and a half hours yourself is just kind of kind of strange um and and and especially since he's got a podcast one of the obvious things to do is to invite somebody on who you know thinks there is a crisis in physics if you don't and he doesn't think there's one it seems and well you could actually have an interesting discussion with this person for for some time but instead of discussing some this it's like you know, there's a controversy going on of two kinds. And instead of inviting somebody on to discuss this controversy with you or two people, you just go on for four hours about how your view that the other side is wrong. It was very odd, I thought. Also, it wasn't as if he was arguing with the people that were saying that there's a crisis in physics so when people say there's a crisis in physics they generally mean that there's a crisis in high energy physics particularly with coming up with fundamental law and so what he was then taking it on to mean is there's a crisis in physics as a whole like cosmology or astrophysics and then he's like no but look in solid state physics and the progress there That's called a straw man where you're not actually taking on the argument. You're taking on a diminished version of it. Well, he was also often involved in these arguments over string theory with me and Lee in 2006. And it was often the same kind of thing that he's kind of... And the whole thing is just odd from beginning to end because he's actually not a string theorist. And this is another weird sociological thing I found is that you find there, you find non-string theorist physics, physicists who somehow want to take a bit aside in this and want to and have a big opinion about it and get emotionally involved in it, even though they actually don't know, don't actually understand the issues. This is not what they do. This is not their expertise. So, and, so I know, I think some of this, you know, knowing, knowing Sean and what he's trying to do, I think he's not the only one who you see this phenomenon, that there are people who, you know, they see what they want to do in the world is really to bring to the public an understanding of the power and the great things that the subject has accomplished. And so even in his four hours, he spends a lot of time, you know, giving very, very good explanations of, you know, various parts of the story of the physics and the history of this. And, you know, they kind of see them, their goal in life is to kind of convince this, you know, the rest of the world who doesn't actually understand these great ideas or doesn't really appreciate them or skeptical about them, you know, to bring them to them. And I think part of, been the whole reason is, I think he was kind of doing this or does this is because, you know, to bring them to them. And I think part of, the whole reason he was kind of doing this or does this is because, you know, having people out there on Twitter or whatever saying, oh, you know, physics sucks, it's got all these problems. It's all wrong, blah, blah, blah, that this is, you know, this is completely against his whole goal in life is to stop this kind of thing and to really get people to appreciate the subject. So I think in kind of a misguided way then enters into this from the point of view of, oh, I have to stop people from saying things about a crisis and physics and get them to really appreciate that this really is a great subject and wonderful subject. And it's, but he kind of that goes too you know, starts to defending things which really aren't defensible and things which he often doesn't really know much about. For instance. Just the details of strength theory. I mean, the reason I wrote this book is that some of these problems of string theory, these questions, you know, people will go on about ADS-CFT and this and blah, blah, blah, blah. This is incredibly technical stuff. It's just, you know, to even understand exactly what these theories are that on both sides of the ADS-CFT thing, what is known about them, what are they, you know, what is the real problem here, what can you calculate, what can you not calculate, what can you not find, what you have to find, what happens other dimensions It's horrendously technical, and very few people actually really know it, but lots of people want to kind of get involved in discussions about it and argue about it without actually understanding actually what's going on. And part of the reason for writing the, not even wrong in the book, but was to try to kind of, you know, to sit down and try to write about about, about, you know, what was really, what was really going on, what the specific technical issues actually were, you know, as much as possible was it in a somewhat non-technical venue. But anyway, so that's some of my reaction to this. And in particular, I mean, he just starts off the whole thing by, he picked up on something from Twitter about somebody had found a paper from somebody written in 1970s complaining about how, you know, there was a crisis, there wasn't any progress in the field. And this was a time when there was great progress in the field. And this was a person who honestly, somebody completely ignorant wrote a completely paper no one ever paid attention to in the mid-1970s that that was wrong about this. And he wanted to use that as to kind of bludgeon people who are making serious arguments about the problems today. So I don't know. I thought it was kind of weird performance. But it is, I think this is a good thing to ask kind of people on this other side of this argument, strictly, why there's very little willingness to actually engage in technical discussions publicly with people they disagree with. I mean, Sean has never invited me to be on his podcast. He hasn't invited to be in a Hassanfelder. It's not, there is no appetite for that at all among people in the subject. And I think, you know, a lot of that is because, you know, they're well aware that, you know, they're really serious, difficult problems with this going. Whether you want to call it a crisis or whatever it is, there are real problems and they're just not very interested kind of acknowledging and publicizing that. Yeah. Well, I have a tremendous appetite for it and the people in the audience of everything do. So if ever you have someone who you feel like would be a great guest with the opposite view that is defending string theory or the state of high energy physics, then please let me know, and I will gladly host you both. Okay. I know we spoke about some people behind the scenes, some people who are likely to say yes and have a congenial conversation. Well, there's actually most people are. I mean, the funny thing is actually early on in this, I was invited, a guy down at University in Florida invited me and Jim Gates to come and debate and debate string theory. And so we, I think we really disappointed this big audience by agreeing on almost everything. So, you know, he's a strong, he's a well-known strength there is. And, and, and, you know, and so we actually found that i think things have been interesting to do this to do this again now but this was almost 20 years ago but let me maybe a little bit less 15 years ago and you know the way i would describe it then is you know if we started talking about the details what our disagreements came down to where it was kind of more, you know, should you be out, you know, we would agree about the state of current things, but where do you think the stuff is going? Are you optimistic? I see reasons why this can't work. He would see reasons why this is actually the best thing to do. He knows how to do and this might work. And there, it's just that kind of, you know, disagreement about ideas, which is, is perfectly reasonable. And actually, Gates told me, I remember at the, at the end of when we were talking after this thing, he said, yeah, you know, I was asked to, like, write a review of your book about it. And I thought, oh, well, I'll just, I'll pick up this book and I'll see, you know, the guy's got it all wrong about string theory and whatever. And then, you know, I read your book and I realized that, you know, a lot of what you were saying was the stuff about, that importance of representation theory in physics and that, and I actually, you know, that's actually exactly the way I see what's important in physics. So I find myself agreeing with much of your point of view and the book. So I couldn't. Anyway, so that was, you know, anyway, at the level of these ideas, I think, especially back then, I think there wasn't, it's perfectly happy, possible to have a reasonable discussion. I think it has become weirder now. You know, 20 years later, they're really, you, I think it was a lot more possible to reasonably be an optimist back 20 years ago and say, well, you know, the LHC is about to turn on. It's going to look for these super partners. Maybe they'll see super partners. There's, you know, we have all this stuff that might vindicate us, and we're all hoping for that. But now, you know, the LHC has looked, the stuff is not there. There's really not, and, you know, that's one thing that's somewhat shocked me is people willing to, people who are often, to me or in public saying, look, you know, the crucial thing is going to be the results for the LHC. You know, we believe that you're going to see, we're going to see these super partners and this is going to show that we're on the right track. And then the results come in and, you know, you're wrong and you just, you just kind of keep going and without even kind of skipping a beat about how, yeah, yeah. Anyway, that's, I think, well, there's a comment on your blog that said, the LHC has just, it's great for string theory because it divides in half the moduli space. Anyway, you can make any kind of joke you want. But, you know, I, that was certainly my feeling a lot when I was writing the book, whatever, is that, you know, this was, this was going to be a crucial thing, this, the LHC, because either the LHC was going to see something along the lines of what these guys were advertising and which they were often willing to kind of actual bet money on, or it wouldn't, and then they would back down and start saying, okay, well, maybe the critics have a point. But, no, I mean, it's just kind of amazing, and people would people will just completely ignore the, you know, the experimental results and keep going. About representation theory, for people who don't know what representation theory is, can you please give them a taste? And then also explain why is it important? More so than say you want a group to act on something. Like, okay, yes, but how much more involved does it get than that well anyway so so just to say that to give a flavor of what we're talking about yeah so i mean it's very common for people to talk about the importance in physics of symmetries and um and when you say that you know that's important to study the symmetries of something, people often then just explain it in terms of a group. So mathematically, a group is just a set with a multiplication operation. You can multiply two elements to get another. But the interesting thing about symmetries really is actually not so much the groups, but the things that groups can act on. So what are the things that can be? So the standard examples like the group of rotations. You can pick things up and rotate them in three-dimensional space, but what are all the things that you can kind of do rotations to? And so those are those in some sense are the representations or the representation theory is kind of the linear version of that theory. And if you try to work with a group action on something, it is a nonlinear, you can look at the functions on it and turn it into a linear problem. But anyway, so group representation theory is really, you know, in, it really is the study of kind of symmetries. What are the possible symmetries of things? What are the possible things that can have symmetries? And it's really fundamental both in physics and it's really, and in mathematics. And I mean, large fractions of mathematics you can put in this language of what are, there is some kind of group and it's acting on some things and what are the representations. You can, I mean, the amazing fact about the language program and number theory is how much of number theory you can formulate in that language. And you can formulate a lot of geometry in this language. It's kind of a unifying language throughout mathematics at a very deep level. But then, to me, the amazing thing is that the same, if you start looking at the structure of quantum mechanics, if you look at what are the quantum mechanics is this weird conceptual structure that states are, state of the world is a vector in a complex vector space and you get information about it by self-adjoined operators acting on this thing. So from the, that looks like a very, very weird, like where did that come from? But if you look at that formalism, it fits very, very naturally into the formalism of group representations. It's really, and this is kind of why I wrote this book, taught this course here and wrote a book about quantum mechanics from that point of view. What's the book called? Quantum Theory Groups and Representations and Introduction. It's kind of a textbook. So it was the second book I wrote. Okay, that link will be in the description. Yeah, and there's also a free version with kind of corrected, with errors that I know about corrected on my website. You can also link to that. No, we want people to pay. They have to pay for the errors. Or you can buy it, or you can buy a copy from Springer if you like a hardcover book or whatever. But, yeah, so anyway, it really is kind of amazing. One of the thing that's most fascinating to me about quantum theory is, you know, that there is a way of thinking about that it's not just some weird out-of-the-blue mathematical conceptual structure that makes no intuitive sense. I mean, it really has a structure which is kind of deeply rooted in understanding representation of understanding certain fundamental symm. Have you heard of this theorem by Radin Moy's in differential geometry about the amount of different structures that can be placed on different dimensions? So for dimension one, there's I think up to diphthism or up to differentiable structure. I forget the exact term. There's just one and then there's just two for dimension two or just one. There's a finite amount for every dimension. Except dimension four. In which case, there's not just an infinite amount. There's an uncountably infinite amount. Yeah. But there's even, yeah, but this is actually, yeah, also one of the most famous open problems in topology, the smooth black array conjecture, which says that, you know, is there, there you're thinking about it, specifically the four manifold, yeah, so is there a, now I forgot what I used to know about this, but yeah, but there are exotic. Well, the point is that dimension four is picked out. And so it would have been nice for physics if dimension four was picked out and finite, whereas the rest were infinite, because then it just means, well, it's nicer for us, but it's picked out and made more diverse and more mysterious. Yeah, but it's, how does this go? Anyway, so anyway, four dimensions is, anyway, topologically, four dimensions is very, very special. Yes. You know, one dimensions and two dimensions, you can kind of pretty easily understand the story is pretty story, the classification story is pretty simple. Three dimensions is harder, but especially with a solution of quackereg conjecture, you could, you actually have a good three-measure classification. And then once you get above four dimensions, things, basically there are more ways to move things. So things simplify so you can actually, you can actually understand above four dimensions what's going on. So four dimensions is kind of a peculiarly complex. Yeah, and so it's, yeah, it's, yeah, it's, but there is, anyway, it's very, I've never actually seen though any kind of clear, clear idea about how, what this has to do with four dimensional, with it, with physics. I mean, yeah, it's, I mean I mean the thing the stuff that I've been doing you know very much crucially involves the fact that four dimensions is special because the um the way spinners work or if you like the the rotation group in in four in every dimensions is a simple group except in four dimensions in four dimensions the rotation group breaks up into two independent pieces and that's at the core of what a lot of what I'm trying to exploit but um so four dimensional geometry is very very special and I don't know speculate very speculative maybe the there's weirdness about infinite numbers of topological structures under four dimensions, that the fact that you've got the rotation group has two different pieces means that is behind that. But I have no, I know, I know, who knows? Of course. Yeah, it's interesting that the fact that it's semi-simple is a positive here. Like you mentioned, it breaks up into two. Whereas usually in physics for the grand unified theories, what you want is simple. You don't want semi-simple. You want to unify into one large group. Yeah. Well, even, there's nothing really in terms of unification. It's just, yeah. Maybe it's a, maybe I should also say something about this about why, what I'm trying to do, I think is quite different than the usual sort of unification and what the usual. Yeah. Yeah, and please explain Euclidean twister theory once more, again, for people who are still like, I've heard the term, I've heard him explain twisters, I somewhat understand twisters, has to do with lines and points and planes, okay, and spinners, something called spinners. I think I understand that. What is Euclidean twister theory? Minkowski's like special relativity. Okay. So they're still confused. Okay. Well, maybe it's better to talk about what other, what standard kind of unification ideas are. And I think, and to my mind, I mean, basically almost essentially all attempts to do the United Founder in the same problem. So one way of stating the problem is we go out and look at the world and we see gravity and we see the electromagnetic interactions and that's kind of based upon a U1 gauge theory, just a circle. We see the weak interactions that are based upon an SU2 gauge theory. That's a three sphere. And we see the strong interactions that are based upon an SU3 gauge theory. So where in the world did this, U1, did these three groups come from, and the way quarks and other elementary particles behave under those groups? So it's a very small amount of group theoretical data. Where did it come from? I mean, why that? And so the standard answer to this very soon after the standard model came about was that, well, there's some big league group. Like you take, like, STU5, take the group of all unitary transformations of five complex dimensions or take the group of all orthogonal transformations of 10 dimensions let's say so 10 and then and then you fit the that that data and show that that data fits inside that bigger structure okay that you can within that s o 10 group we i can fit u1 and suU2 and the SU3. You can get them in there. And then I can put all of the known particles together with their transformation properties and give them and make them have a, and put those together as a transformation property of S-O-10. So you can kind of put stuff, this kind of package of algebraic data we're trying to understand where it came from. You can put it together in a simple group into a group where the problem is in terms of group theory, it's a package involving several different groups. And so you get several different simple groups. So you can you can anyway you can put this together but but the problem with this is always is if you try and do this you can then write down your SU5 or S010 theory or whatever and and you know it looks a lot nicer than the standard model it's only got one one term where you had a lot of terms before but you have to then explain but wait a minute why don't we see that why do we you know why do we see this this more complicated thing and not that and so for instance the standard thing that grand unified theories do is they you've put the weak the weak interactions and the strong interactions into the same structure so you should have, anyway, so all sorts of things, there are all sorts of new kind of forces that you're going to get in this bigger structure, which are highly constrained, which have to exist, which are going to do things like cause protons to decay. So like, you know, why? Sure, sure. Yeah, so you put the stuff together, all of a sudden, it can interact with itself and it can do things which you know don't happen, and protons don't decay. So your problem, when you write down these theories, the problem is you haven't necessarily done anything. You've put the stuff together in something bigger, but you haven't, you've just changed the problem from why, you know, why, why these pieces to, to why did this bigger thing break into the, how, how do, why did this bigger thing break into these pieces? You haven't actually solved until you have an explanation for that, you haven't actually solved anything. And this is, I think, the fundamental problem with these grand unified theories. They don't come with a really, the only way to make them break down into these other things is to introduce more Higgs particles and more complicated structure and more degrees and more numbers. And you lose predictivity if you do that. You also find that they also don't look like what you see in the world if you do experiments. But most people who have tried to come up with some unification have done some version of that actually. I mean, so for instance, I mean, I don't want to really get into things like what Garrett Leasy is talking about you know, they, they, they, they've all got their own version of this. And I think when you see people kind of dismissing theories of everything and green and fight theories and you see, um, Sabina Hassanfelder are saying, well, you know, these people are lost in math, then they're, they're, they're all really referring to the same problem that people are trying to get a better understanding what's going on by putting things together into a bigger structure and then and they're all kind of foundering on not having an answer as to why why this breaks up so um so the thing that i'm trying to do it that why i much for interested in these ideas about spinners and twisters, is that I'm not actually, I mean, a lot of what I'm doing, as I said, I mean, the fact that there are these two SU2s, that's an aspect of four dimensions. There really are, maybe the thing to say is that I'm not, I'm not introducing kind of new, I'm not introducing lots of new degrees of freedom and then having to explain why you can't see them. I'm trying to write down something. I'm trying to write down a new geometrical package, which packages together the, the things we know about and doesn't actually have new, you know, doesn't actually have all sorts of new stuff. Penrose said this was his motivation as well for Twister theory. Yeah. Yeah, so Twister theory, so in some sense, twister theory is a bigger structure, but it's not, it doesn't kind of contain anything really new. It contains the same spinner as you had before and puts them in an interesting new relation so you can understand conformal invariants. But he doesn't, it's like, you know, twister theory is not the things you knew about twister theory. It's not spinners and vectors of the things you knew about plus some other completely unrelated stuff. It's the things you knew about in a new, more powerful conceptual framework. And so that's the sort of thing I'm trying to do. Part of the problem is that, you know, it's, I guess a misnomer to really say this is a well-defined theory. It's more a speculative set of ideas about how to, but that's's the crucial i mean probably i think the most important new idea here which the which for this to be right has to be true and which is something is exactly this idea about um about rotate that if you think about rotations in four dimensions and euclidean space time when you relate it to to Mankowski space time in the real world, one of the SE2s can be treated as an internal symmetry. And that could explain the weak interactions. That's kind of a crucial. That's why it's also referred to as gravel weak unification by you or by other people? Well, other people have noticed this. And actually, it's interesting when you read the literature on Twister theory, people point this out, they say exactly the problem I was pointing out that this is a very chiral, chirally asymmetric view of the world. And a lot of people said, oh, well, that means, you know, maybe you should be able to understand, you know, the weak interactions are chirally asymmetric, so maybe there's something here. But the twister people, I think, never really had a version of this. I mean, there are various people who have tried to write down to do this. I mean, one is actually, there's a paper by, you know, Stefan Alexander has worked on this and Lee Smollin. They actually had a paper attempt to do this. But they, I mean, what they're doing is significantly different than what I'm trying to do. In particular, they're staying in Minkowski space. I mean, this idea of going to Euclidean space to get the, anyway, to get this thing to behave like an internal symmetry is not something that isn't their work. I know. You know Jonathan Oppenheim? A little bit, yeah. I mean, I've known. Yeah. Jonathan Oppenheim, Stefan Alexander, and Nima Arkani-Hamed all were graduate school peers at the same time as my brother in physics. Oh, okay. This is interesting because then later on in my life. This was all in Canada, right? Yeah, yeah. So UFT, Nima was at UFT, University of Toronto with my brother, but then in graduate school, Oppenheim, Stefan Alexander. I spoke to Stefan on the podcast as well. Yeah, no, so there have been very few physicists who have been encouraging about this. So he's one example. Yeah, he's extremely open to new ideas. And playful. He's a playful person with that, much like with his music. I think that both qualities rub off on one another. And I think also in his own research, he's also, I think he hasn't, it's not so much that he's followed up on this Grave-A-week stuff, but he's, he is very interested in, you know, is there some way in which gravity, you know, that gravity actually is a chiral theory. There is some chiral asymmetry in gravity. And especially, you know, can you know, anyway, I mean, are there kind of astrophysical and cosmological places you can go and look and see, you know, is gravity really, chirally symmetric or not? And so I know that that's something that he's worked a lot of. So he's working on experimental tests of the chirality of gravity, but that doesn't mean experimental tests of your theory, just your theory is a chiral theory of gravity. Yeah, it's a, it's a chiral theory. But it's not, it would be validation of your theory or a testation? No, I mean, it's kind of, I mean, first of all, again, I have to keep thinking, I don't really have it. I don't, I would love to say, I would love to say I've written down a consistent proposal for a theory of quantum gravity based on my ideas but I'm not there yet. And I think what he's doing is more, it doesn't involve, doesn't have, the structures I'm trying to exploit are not there in what he's doing. But I believe what he's doing is more kind of thing. You kind of add Chern-Simon's kind of terms. You assume that maybe there's some Chern-Simon's term in the theory and ask, you know, what the observational implications of that would be and try and go out and look for that. But I haven't looked, I haven't really carefully looked at what he's doing, just because it's quite different than what I'm trying to do. Can you explain what Turn Simons theory is? So what it means means to add a Churns Simon's term. I know Stefan's worked on Churns Simon modified gravity. And then there's something like Churns Simon terms in the Lagrangian of particle physics, but I don't know if those two are related. Yeah, I don't, yeah, I shouldn't try to talk about it as work as I don't remember exactly what he was doing. But, well, Churn's time in the, it's very hard. Actually, one funny thing is that I actually went to, I don't know, so I actually started thinking about churn. So maybe I can go back to, you know, how I first encountered them. So when I was doing my PhD thesis, my problem was I'm trying to understand, I've got engaged on a computer, and I've got this version of gauge fields, and they're described on links on a lattice and you can store them in a computer and manipulate them. And I want to look at one of these configurations and say, you know, there's supposed to be some, there's some interesting topology in this engage theory. And this is what people are getting interested in the 70s and 80s. And so in particular, there's something called the, let's say the instanton number. And so, you know, these gauge fields are supposed to have some integer invariant called the instanton number. And if somebody hands you a gauge field on a compact manifold, you should be able to calculate its instanton number. And you can then, then you could, if you could measure these, if you could calculate these instanton numbers and see them, you could do interesting physics with it. So the problem in some of this problem my thesis was, you've got these gauge fields, what are their instanton numbers? Can you define them? And so... And they're just integers? They're just integers, yeah. So they're invariants, they're not invariance of the base manifold. You basically have a bundle with connection and they're invariance of the bundle. And if you know the connection, you're you're sensitive to this invariant. But the one way of looking at that though is if you look at the integral formula for this thing, it's a total derivative so that if you try to integrate it over a ball or a hypercube, the formula that's supposed to add up to this instanton number, you can write it as an interval with the boundary, right? It's the interval of D of something, so it's the, it's the integral of boundary. It's a total derivative, so you can see. So the, so the thing that it's a total derivative, the thing that that lives on the boundary is the is the Chern Simons form. Okay. So that's, this is kind of the first way that people started seeing this thing in, in physics is that. And so, so, so one idea was I, well, I could, um, I could um yeah if I could call instead of calculating these instant on numbers if I try and do it in terms of their local contributions from each hypercube I should if I could just calculate the churn simons not the churn simons number the contribution you know the if I could cut could cut that the that thing then then i would be done and so i spent a lot of time looking at the churn simon's formula and and then i spent a lot of time trying to put that in the lattice and then i kind of finally realized it's kind of gauge the problem is that it's very gauged in variance so any kind of idea you have about how to calculate it or construct it tends to be just an artifact of some choices you're making because of gauge symmetry. So this, though, that led to one of the great experiences of my life. When I was at a MSRI, you know, Atia was visiting and at one point Atia and a bunch of people were talking to the blackboard and somebody was asking Atia said, oh, you know, how would you like in, you know, how would you calculate this churn Simons network? Then churn Simons had become incredibly important because of Witten and and so everybody was like, Witten had said, you can get these wonderful nod invariants of three-metapult of variance if you can do path integrals and that you should take the path integral to be E to the I times the churn-Simon's number. Exactly that integral that I was talking about. Yes. But Witten now wants to integrate it over a whole three-manifold. And so people were asking, Atia, well, you know, can we try and think about how could we actually do this calculation, what were we doing? And so, and then Atia, for thinking for about for about five seconds, comes up and says, oh, well, maybe, you know, you could calculate it this, you could calculate it this way, do this. I was luckily standing there. And since Atia had thought about it for about 10 seconds, I thought about it for about three years. I could say, no, no, no, that doesn't work. You can't do that because of this. Oh, great, great, great. So that was one of the high points of my mathematical career. Yeah. Anyway, but I don't know that this is in any way answered any question, but that's one definition of it. But it's a very, it's kind of an amazing piece of information about, you know, about gauge fields, about connections. And it tells you some very subtle things. And it turns out to be useful for all, describe all sorts of interesting and unexpected physical phenomena. And these speculative ideas of yours of gravel weak unification, have they been sent to Penrose? Has Penrose commented on them? I haven't heard anything back from Penrose. Predros is a little bit of a problem that I don't actually... Anyway, whatever email I had from him back when he was helping my book no longer works and other emails tend to bounce and say... You don't have mutual friends? I could make more of it. I haven't made more of it. I also keep also hoping... I've come this close to actually running into him and being at the same conference and something at him and having a chance to talk to him personally, I keep expecting, instead of making a further effort get to get a manuscript to him part of the problem you'll see if you try and if you don't know his email and you try and contact them you end up getting a secretary and who may or may not see more anything to him right but I keep hoping yeah I was actually at Oxford last year and actually was there somebody who showed me oh, oh, that's Penrose's office. And then I went to do something else. And then the next day, they said, oh, you know, 15 minutes after we were there, Pedro showed up. Oh, boy. The lowest points of your mathematical career. Well, I don't know. I don't know how this would work. From things that he said about this kind of thing, I think he's made it very clear that he has always explicitly, he's been, you know, he's followed the kind of thing Atia did, the kind of Euclidean version of the theory. But he's always said very clearly that in his mind, the Euclidean version of theory is not the theory. What's,'s happening in Mokowski space. And so he's, anyway, whether I could convince him otherwise, I don't know. But I think he's kind of pretty clearly in his mind thought through, okay, there is this interesting Euclidean theory, but that's actually not really the physical thing is Mekowski. So I don't actually believe you're going to, that by working over there, you're going to actually tell me something important. But I think I'd have to get around that particular initial reaction from him. So forgive this fairly foolish question, but if both GR and the standard model can be formulated in terms of bundles, then why can't you just take a direct product of the groups? So, for instance, you have the standard model gauge groups, and then you direct product with S.O.13. So that's the principle, and you make an associated frame bundle. That's like just the projection of S.O.13. And then you say that's general relativity, and the other ones, the other associated bundles of the standard model. And then you call that unification. Is that unification? What are the problems there? Well, the problem is that general relativity is a different. Well, maybe the thing to say is, so gauge theory is really just what you have is a bundle and the fibers are some group. And you have connections and curvature on that. You write down the interesting Lagrangian is the norm squared of the curvature. And anyway, so Gage series is a nice pretty story. If you try and write generatively the same language, you know, you can do it. It's fine. You have a G bundle where G is S.O3-1 or the Euclidean Ridge, whatever. Yeah, yeah. And you have a connection, you have a curvature. But the problem is that you crucially, the problem is that you crucially have something else and you have other things specifically because you're not some arbitrary G bundle, you're the frame bundle. And the frame bundle, you know, it has, you know, it's a principal bundle for, you know, the group of just all changes a frame. But it also is, I mean, people use the term soldered or tie. It's also, it also knows about the base structure. So a point in the fiber of the frame bundle is not just an abstract group element. It's a frame. It's a frame down on, you know, if you can take vectors, you can protect it on the base space, and it's a frame for those vectors. So it's kind of soldered to the tangent space. it, what, what this means in practice is it means that there's, there's, there's new, there's new variables which are in the, which you have, which, which are part of the story, which are not just the, not just the S.O3-1 connection and curvature. There's also, you know, so you've got this connection one form and cur. Sodering form? Yeah, it's called the soldering form or the tetrad or, I mean, there are a lot of different people have names for it. But there's kind of, there's kind of a one form you feed at the vector and you feed it a vector and it tells you and, you know, since you're up in the frame bundle, you've got a frame and this one form has, you know, it has you, and since you're up in the frame bundle, you've got a frame, and this one form has, you know, it has components which tell you what the components of the vector are with respect to the frame. So it's a very kind of canonical object, but, you know, it's there. The space-time geometry depends upon it. So the space-time geometry doesn't just depend upon the connection, the curvature, depends upon the connection, the connection, and this, this, this, this, this, this, this, this, this, this, this, this, this, this, um, this canonical one form. So, so, so, so it, you, you've got extra variables, which you didn't have in the, these just don't exist in the Yang Millsills case and you have to and so you can and and with those variables you can you can write down a different look a different lower order of Lagrangian instead of taking the taking the curvature squared you can take the curvature times some of these guys and you can get the Einstein Hilbert Lagrangian sorangian. So the fundamental Lagrangian of gravity is very different than the fundamental Lagrangian of Yang Mills theory. And it's because you've got these extra gadgets to work with. I see, I see. They've got a one form. So that's one way of saying it. You can't. But people have speculated a lot about why, you know, why not, why not just try, like, adding these higher curvature terms like you had in the, in the Yang Mills case, add those to gravity. And anyway, there's a long, long story about trying to mess with different change the Lagrangian of gravity to try to something better behaved. Now, have you found any unification attempts that are between gravity in the standard model or gravity in any of the interactions that are improved if you don't view gravity as curvature but rather as torsion? So, for instance, this is something Einstein was working on later in his life. And then there's also non-matricity. Carton was working on that. Yeah. Yeah. And they're equivalent formulations of gravity, at least the torsion one. The gravity is actually not curvature, it's just torsion. Yeah, yeah. So the, well, one way to say it is, so now once you've got these, so the thing about, if you write, start writing down a theory of gravity. Well, first of all, I mean, non-metricity, I think some of that may just mean, actually I'm not sure what exactly the people mean about that. I shouldn't say. So the two compatibility conditions to create the Levi-Cavita connection, I believe it's called, is that you have no torsion and that you have that the metric doesn't change with the covariant derivative. So if you take the covariant derivative on the metric, it's zero. If you don't have that, then you have non-metricity. In other words, along the parallel transport, the metric is preserved. Yeah, okay, yeah. I'm not so sure about that. But I can't say about torsion, that the, but your problem is that if you, so if you just write down a theory with with some you put together a Lagrangean which is which is going to be give you equivalent results to the Einstein Helbert you put it together out of the curvature and the canonical one form now your problem is that you've got you know when you try to get the other Lagrange equations you you can, you can vary the canonical one form and you can vary the connection. So you've got, and one of them, let's say, I guess it's, if you vary the connection, then you end up, that gives you the torsion free condition. So, so, so, so, so, so you, you've got more variables, so you need more equations. So you recover gravity, but you recover with the standard Lagrangian, you recover not the Einstein's equations and as one equation, but also the torsion-free condition as the other one. So I mean mean, so the standard simplest, you know, version of Einstein-Hilbert in that theory, you know, has no torsion again. But you can certainly write down more different Lagrangians in which torsion is, you know, is not zero, but it's some kind of, has some kind of dynamics and does something. And that might be interesting. Yeah, I was watching a talk a few, maybe a few weeks ago or a couple months ago about when trying to modify gravity, especially for explaining quote unquote dark matter that you can explain dark matter as a particle, but if you want to do modified gravity, it's useful to have torsion in your theory. Well, anyway, what I was thinking was, okay, if it's useful there, maybe it's not actually the case that that explains dark batter, but maybe it would be more useful to try unification with torsion models of gravity than with the regular curvature model of gravity. Yeah, I should say one kind of funny thing about all this is that I've always, I mean, before I got involved in this particular thing, I tended to kind of stick to thinking, I mean, I spent a lot of time over the years trying to learn about quantum gravity and about these issues that we're talking about. But I never actually, you know, got really serious about them and developed any real expertise with them because I always kind of felt that they're, I don't know, I'm trying to understand what's going on in particle physics and the standard model. And there's, there are these groups of people who, you know, just think, who just think about quantum gravity and that, you know, they're very smart. They've been doing this for 30 or 40 years 40 years and even and a lot of them aren't strength there is and um and you know i don't i'm not seeing anything that they're doing that i that or that i could have any kind of you know that i could do it anyway better like you know that they seem to be doing interesting things with torsion but they know more about torsion than i don't do so right yeah so i i kind of, anyway, I kind of stayed away from a more particle. Yeah. Yeah, exactly. Yeah, that's the way of saying it. But I really stayed away from kind of going more in that direction, becoming more expert, a lot of these things, figuring, yeah, I mean, until I see something that I could, that maybe I can do something with, I mean, if it's just, it's interesting to see what the story is there, but they're really smart people who have been banging away at the story for a long time, and I can't help. I'll stay away from it. But, so yeah, so I kind of have the, I've actually partly because of this had to, had to learn a lot more about, it gets some remedial education on some of this stuff. And so I'm, but I'm still in some sense the wrong person to talk to about theories of gravity and about the... Yeah. Before we wrap up, there are a couple other proposed toes, so one with Lisi, like you mentioned. And then Eric Weinstein has Geometric Unity, and Wolfram has Wolfram's Physics Project. I believe that's still the title. And Cheramar-Marletto has a framework, not an actual toe, but construct your theory. So which of those have you delved even superficially into? And what are your comments on them? I should say, I mean, the Wolfram or the other one mentioned, so these ideas that you're going to start with some completely different starting point like Wolfram. We're going to start, I don't know, whatever you want to call, whatever he's starting with. The fact that you're going to start from this kind of completely different thing, it has nothing to do with any of the mathematics that we know of, and that you're going to then reproduce the standard model, whatever this. That seems to be highly implausible. Anything I've ever looked at, and of his for briefly, you know, doesn't change that opinion. I just, I just don't see how you get from. Anyway, I mean, you're telling me that you're going to go and start way, way, way, far away at something else and make some progress right here. And I don't see how you're going to get, you're ever going to're ever gonna get back and so so there's a lot of that um uh leesiest thing i looked a bit out a bit so i i know garrett and eric both fairly well you know so garret has slept on my couch like many people but uh and and and you know so garret i think you had well-defined proposal, but to my mind, it has exactly the same, the problems that I was telling you about. You know, he, he wants to put. So these are the same problems you explicated about Grand Unified theories earlier. Yeah. So he wants to put all these things together, and he wants to put it together and have it live inside E8, and it's very nice, except that he doesn't really have a, to my mind, by doing that, he hasn't actually solved the problem. He has to tell me why the E8 breaks down into the pieces that we know about. And he doesn't have any, as far as I know, has no useful idea about that. But he is a fairly well-defined thing. I mean, Eric, you know, I've talked to a lot about this over the years. I don't know. I mean, he, and I've looked a bit at, you know, paper that he finally put out. But I think, again, it seems to me, it has the same kind of problems. Again, he's trying to put, he's trying to put everything together into this bigger geometric structure. But he doesn't, to my mind, have any kind of plausible idea about how he's ever going to break that down and recover what we, the real world that we see. And his is a lot harder to see exactly what he's doing or unless Lizzie is kind of following much more kind of a standard story. You can see exactly what he's doing where it's harder to tell. But both of them, I think, suffer from the same problem as guts as far as I know. What about category theory? There's plenty of hype about category theory in physics, but you're also in math, and so you're much more close to category theory. Is there a hope that somehow higher categorical structures will elucidate how to make progress in high-energy physics? Yeah, I haven't seen any evidence for that. I mean, the things people are doing with those are actually much more trying to understand. There's a lot of people actively trying to use some of that mathematics to understand like classification or more kind of theories you would use in condensed matter systems. So it's possible that, you know, the right way to understand, you know, gauge groups, you know, the infinite dimensional group of all gauge transformations, or you're even, or maybe you can even think of the diphthymorphism group about how to think about representations of those groups, those groups, and maybe that the higher categorical stuff has something useful to say about that, because there are the problem is that you, the standard notions of what a representation is don't really, the problem is when you're dealing with these influential groups, you really don't even know what, you can't just say representation, you have to put some more additional structure to make this well defined and what the additional structure is unclear and maybe it would help with those. But anyway, I haven't really followed. I've spent some effort trying to follow that mathematics, but I don't do that. Anyway, category theory in general is just a very, very general idea. The problem is it's a very, very general idea. So it's something, it's part of, you know, the way mathematicians think about every subject, you know, that I really, it's very, very useful to think not about representations, but the category of all representations to think of, and that opens up all sorts of new, quite new ways of thinking and questions do that, but it's, but it, it's just a very rare abstract language. So it can be used for many, many things. And I think when I realized at some point, when I was a student, I was very, I thought, okay, well, you know, the way to understand mathematics is to find, you know, look at these, the mathematics are teaching us and look for the more and more general structures and then just find them, understand the most general structure. And then, you know, you'll be able to, to derive the rest of this stuff. And so, and then it looked like category theory was that was this thing, which was the most general thing that people were using. And so I thought I should go learn category theory. But then at some point, I realized that what I was, what you're doing is that as you go to greater and greater generality, you're, you're saying what you're doing, you're talking about, you're saying something about more things, but you're saying less and less. And so in the limit, you're saying nothing about everything, which is really not, not actually a useful limit. And that's the problem with just, you know, category theory has just in its most general meaning. It's very useful. I can do all sorts of things, but it's not, anyway, it's telling you a bit about everything, but yeah, it's too much generality to really kind of. Now, what if someone retorts about the polemics against string theory by saying, hey, look, string theory has produced something much that's positive. So, for instance, the math is used in condensed... Sorry, is used in the fractional quantum hall effect and many other condensed matter systems. No. That's, yeah, no, the string theory hasn't... That stuff doesn't... Well. First of all, I mean, a lot of the time when people are talking about this, they're talking about something which didn't actually come from a string theory. It's quantum field theory. So yeah, like the fractional quantum whole effect. I mean, I don't think there's not a string theory. There was a comment that said, look, I'm a physicist and I'm not a string theorist, but we use string theory in the fractional quantum hall effect. And that was a comment on the Ed Frankel video. Well, I think probably, I mean, the problem is string theorists are happy to kind of claim, yeah. Anyway, I mean, they're kind of claiming that everything comes from a string theory. And they're actually at this point, David Gross kind of argues that, well, you can't, you have to shut up and stop arguing about string theory because string theory and quantum field theory are actually all one big thing. And so you're arguing against quantum field theory. So that's just a ways that. Because string theory is supposed to be a generalization of quantum field theory? Well, it's because, oh, you know, with these dualities and M theory, whenever we realize it's all the same. And so anyway, so I don't know in this, in this specific case, and I'm not an expert on that case, but I strongly suspect that the saying that this came from string theory is that it's really some fact that they learn from string theories. And string theor is happy to say this camera of string theory, but it's not actually. And to make this whole thing even more frustrating, more complicated, is that no one actually can, at this point, has a definition of what string theory is. So you can, people then start talking about kind of like what Gross is trying to do. He's trying to say, well, string theory and quantum field theory all the same. So when I say string theory, I mean quantum field theory. And people just keep doing this. And, you know, so we, anyway, unless you're really, really expert and you know exactly what the story is about what string theory is and how it's related to quantum field theories, whatever, you easily get very confused. Another weird thing I found is that almost everyone believes that Ed Witten wrote one Fields Medal for his work on string theory, which is just not true. It's just not true. I mean, the things that he won the Fields Medal for are these totally amazing things in mathematics are actually quantum field theory. Things are not. They actually have basically nothing to do with string theory. The positive energy theorem. Yeah. And those things, I mean, they're not string theory. But, you know, it's really hard to convince anyone of this. Even most mathematicians believe this. If you go up and ask a mathematician, you know, did Witten, a string theory part of what Witten won the Hill's Melbourne? I'm sure the walls. Most of them will say, oh, probably is. Yeah, it sounds right. So what's a fulfilling life for you, Peter? Well, I'm very, I'm quite happy. I mean, one, I think, you know, when my book came out, a lot of people, you know, kind of the ad hominem attack was, oh, here's this guy who was not a success and didn't really, and he's just embittered and unhappy. And they didn't realize that I'm actually quite, quite disgustingly pleased my life and very happy with myself. And things that have gone. I mean, had a weird career here at Columbia and it's a it's a very but I've been extremely well treated by the department and allowed pretty much to do to get away as I said get away with doing whatever I want and treated well and paid well and had a very pretty very happy life and so I'm meaningful yeah and I'm I'm proud of the books I've written, some of the things I've done. And I'm actually quite excited about what I'm working on now. I mean, and this was always one of my great frustrations is that, you know, there were a lot of things that seem to be that something interesting was going on, but I didn't understand enough to really be sure this is really something, you know, I've really got something here. And now I'm much more optimistic about that. And so I'm trying to, I'm getting older though. I'm 66. I'm trying to figure out, I'm actually trying to negotiate with the department of the university, some kind of exit strategy out of my current position to some different kind of situation here. And I may, where I might be doing less teaching and less to, and, and less involved and less taking care of the computers, get other people to do that. So we'll, we'll take care of the computers. Well, I told you about this. So part of my, my, I'm, my official title is senior lecturer. And the weird thing about this title is, is this is a title that the university gives to people who are, they're non-tenured positions, but are, but are teaching, teaching courses here. And so I'm doing that. But I've also, part of the deal with the department has always been that I do relatively not that much teaching, but also make sure the department computer system runs. And so I actually do, on a day, day basis, I also make sure our computer system's going. So I do. You don't want to do that anymore. Well, let's just say I like to do, maybe a better way of saying it is, I mean, I've actually actually kind of enjoy that actually. That's always been never, that's always's always been been been in some ways fun but um there there is an inconsistency i found between you know having the time and focus to work on making progress on the stuff i want to make progress on and also teaching a course and also having to deal off and on with computer problems. And trying to fit all those together in a 40-hour week is not really, doesn't work so well. And I've decided in my life, I definitely have to prioritize the working on these new ideas. I've got to start dumping some of the other things and change things. But we'll see. I managed to find that specific comment that was referenced earlier, and I sent it to Peter Woite over email. Here's the comment, and then subsequently there'll be Peter's response. I am a physicist, and I use string theory all the time in my research on the fractional quantum hall effect. What Frankel means here is that the expectation to find the standard model in the 90s, by Calibiaw-compactification of one of the the super string theories turned out to be unfulfivable to this date. This does not harm the theory. The prediction was just wrong. Therefore, the title of this video is misleading. String theory revolutionized the way we understand physics and math in general, and it continues to do so. By the way, it's the only consistent theory, unifying quantum field theory and gravity. Peter's response is, hi, Kurt. In the podcast, I misunderstood what you were telling me that a condensed matter theorist was saying that they thought understanding the fractional quantum hall effect used string theory. I was speculating that they were misunderstanding some QFT explanation as a string theory explanation. It seems, though, that this is not a condensed matter theorist, but a string theorist. The quote-unquote string theory revolutionized the way we understand physics and math in general and continues to do so is just pure hype. It's the sort of thing you will ever hear from a string theorist devoted to that cause. I was unaware that some string theorists have worked on embedding the fractional quantum hall effect system in a complicated string theory setup. I don't understand the details of this from long experience, think it's highly likely. This, like many, string theory explains condensed matter physics claims, is just hype. String theory since the beginning has had a huge problem, and it continues to this day. The current tactic for dealing with the failure of string theory hype around particle physics is to double down with new hype about nuclear physics, condensed matter physics, and quantum information theory, etc, etc. Peter then quickly sent a follow-up email, hey, I just read the thread. I'm guessing this is a string theory undergrad or graduate student. The claims about the fractional quantum hall effect are based on relating it to Chern-Simon's theory, which is a QFT story, so a quantum field theoretic story. Also, all those fans of David Hesteens should know that I did ask Peter about geometric algebra, but he's not familiar enough to comment on it. Okay, well, it was wonderful speaking with you, and I hope we speak again. I hope we meet in person. Oh, sure. Let me know if you're ever in New York. Oh, yeah, I go quite frequently, so I'll let you know the next time I'm there, and maybe I'll see you at perimeter if you ever come down this way. Yeah, I haven't been there yet, but I would at some point like to like to go there. I just signed up to participated via Zoom. They have a conference on quantum gravity at the end of the month. But it's mostly virtual. And so you can anyway, I'll watch some of the talks on Zoom, but someday I'll actually get there physically. All right, sir, take care. Okay, thanks. Thank you for coming on. Bye now. Bye, bye. The podcast is now concluded. Thank you for watching. If you haven't subscribed or clicked that like button, now would be a great time to do so, as each subscribe and like helps YouTube push this content to more people. You should also know that there's a remarkably active Discord and subreddit for theories of everything where people explicate toes, disagree respectfully about theories and build as a community our own toes. Links to both are in the description. Also, I recently found out that external links count plenty toward the algorithm, which means that when you share on Twitter, on Facebook, on Reddit, etc., it shows YouTube that people are talking about this outside of YouTube, which in turn greatly aids the distribution on YouTube as well. Last but not least, you should know that this podcast is on iTunes, it's on Spotify, it's on every one of the audio platforms, just type in theories of everything and you'll find it. Often I gain from re-watching lectures and podcasts and I read that in the comments. Hey, toll listeners also gain from replaying. So how about instead re-listening on those platforms? iTunes, Spotify, Google Podcasts, whichever podcast catcher you use. If you'd like to support more conversations like this, then do consider visiting patreon.com slash kurt Jymungle and donating with whatever you like. Again, it's support from the sponsors and you that allow me to work on tow full time. You get early access to add free audio episodes there as well. For instance, this episode was released a few days earlier. Every dollar helps far more than you think. Either way, your viewership is generosity enough."""] + + def run_client_chat_stream_langchain_fake_embeddings(data_kind, base_model, local_server, inference_server, simple=False, chat=True): t0 = time.time() diff --git a/tests/test_langchain_units.py b/tests/test_langchain_units.py index 2bd39b2c2..81b1be9ca 100644 --- a/tests/test_langchain_units.py +++ b/tests/test_langchain_units.py @@ -10,7 +10,7 @@ from src.gen import get_model_retry from tests.test_client_calls import texts_helium1, texts_helium2, texts_helium3, texts_helium4, texts_helium5, \ - texts_simple + texts_simple, texts_long from tests.utils import wrap_test_forked, kill_weaviate, make_user_path_test from src.enums import DocumentSubset, LangChainAction, LangChainMode, LangChainTypes, DocumentChoice, \ docs_joiner_default, docs_token_handling_default, db_types, db_types_full @@ -1990,6 +1990,9 @@ def get_userid_auth_fake(requests_state1, auth_filename=None, auth_access=None, assert assert1 or assert2 +@pytest.mark.parametrize("max_input_tokens", [ + 1024, None +]) @pytest.mark.parametrize("data_kind", [ 'simple', 'helium1', @@ -1997,11 +2000,13 @@ def get_userid_auth_fake(requests_state1, auth_filename=None, auth_access=None, 'helium3', 'helium4', 'helium5', + 'long', ]) @wrap_test_forked -def test_merge_docs(data_kind): +def test_merge_docs(data_kind, max_input_tokens): model_max_length = 4096 - max_input_tokens = 1024 + if max_input_tokens is None: + max_input_tokens = model_max_length - 512 docs_joiner = docs_joiner_default docs_token_handling = docs_token_handling_default tokenizer = FakeTokenizer(model_max_length=model_max_length) @@ -2019,6 +2024,8 @@ def test_merge_docs(data_kind): texts = texts_helium4 elif data_kind == 'helium5': texts = texts_helium5 + elif data_kind == 'long': + texts = texts_long else: raise RuntimeError("BAD") @@ -2035,22 +2042,40 @@ def test_merge_docs(data_kind): if data_kind == 'simple': assert len(docs_with_score_new) == 1 - assert all([x < max_input_tokens for x in tokens]) + assert all([x <= max_input_tokens for x in tokens]) elif data_kind == 'helium1': - assert len(docs_with_score_new) == 4 - assert all([x < max_input_tokens for x in tokens]) + assert len(docs_with_score_new) == 4 if max_input_tokens == 1024 else 2 + assert all([x <= max_input_tokens for x in tokens]) elif data_kind == 'helium2': - assert len(docs_with_score_new) == 8 - assert all([x < max_input_tokens for x in tokens]) + assert len(docs_with_score_new) == 8 if max_input_tokens == 1024 else 3 + assert all([x <= max_input_tokens for x in tokens]) elif data_kind == 'helium3': - assert len(docs_with_score_new) == 5 - assert all([x < max_input_tokens for x in tokens]) + assert len(docs_with_score_new) == 5 if max_input_tokens == 1024 else 2 + assert all([x <= max_input_tokens for x in tokens]) elif data_kind == 'helium4': - assert len(docs_with_score_new) == 5 - assert all([x < max_input_tokens for x in tokens]) + assert len(docs_with_score_new) == 5 if max_input_tokens == 1024 else 2 + assert all([x <= max_input_tokens for x in tokens]) elif data_kind == 'helium5': - assert len(docs_with_score_new) == 3 - assert all([x < max_input_tokens for x in tokens]) + assert len(docs_with_score_new) == 3 if max_input_tokens == 1024 else 1 + assert all([x <= max_input_tokens for x in tokens]) + elif data_kind == 'long': + assert len(docs_with_score_new) == 6 if max_input_tokens == 1024 else 6 + assert all([x <= max_input_tokens for x in tokens]) + + +@wrap_test_forked +def test_split_and_merge(): + kwargs = {'max_input_tokens': 7118, 'docs_token_handling': 'split_or_merge', 'joiner': '\n\n', 'non_doc_prompt': '<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nGive a summary that is well-structured yet concise.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n"""\n\n"""\nWrite a summary for a physics Ph.D. and assistant professor in physics doing astrophysics, identifying key points of interest.<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n', + 'verbose': False} + from transformers import AutoTokenizer + tokenizer = AutoTokenizer.from_pretrained('meta-llama/Meta-Llama-3-8B-Instruct') + from langchain_core.documents import Document + docs_with_score = [(Document(page_content=page_content, metadata={"source": "%d" % pi}), 1.0) for pi, page_content in enumerate(texts_long)] + + docs_with_score, max_doc_tokens = split_merge_docs(docs_with_score, + tokenizer, + **kwargs) + assert len(docs_with_score) == 6 @wrap_test_forked From e65cd157863e45ef71516e62d064185d4ccade2a Mon Sep 17 00:00:00 2001 From: "Jonathan C. McKinney" Date: Fri, 10 May 2024 23:46:29 -0700 Subject: [PATCH 2/9] Generalize split_merge_docs to handle semantic splitting (if GPU), sentence splitting, and multilingual --- src/gpt_langchain.py | 97 ++++++++++++++++++++++++++++++++------------ src/version.py | 2 +- 2 files changed, 73 insertions(+), 26 deletions(-) diff --git a/src/gpt_langchain.py b/src/gpt_langchain.py index 1067a736a..99d4dd13a 100644 --- a/src/gpt_langchain.py +++ b/src/gpt_langchain.py @@ -6790,14 +6790,15 @@ def select_docs_with_score(docs_with_score, top_k_docs, one_doc_size): class H2OCharacterTextSplitter(RecursiveCharacterTextSplitter): def __init__( - self, - separators: Optional[List[str]] = None, - keep_separator: bool = True, - is_separator_regex: bool = False, - **kwargs: Any, + self, + separators: Optional[List[str]] = None, + keep_separator: bool = True, + is_separator_regex: bool = False, + **kwargs: Any, ) -> None: """Create a new TextSplitter.""" - super().__init__(separators=separators, keep_separator=keep_separator, is_separator_regex=is_separator_regex, **kwargs) + super().__init__(separators=separators, keep_separator=keep_separator, is_separator_regex=is_separator_regex, + **kwargs) self._separators = separators or ["\n\n", "\n", " ", " ", ""] @classmethod @@ -6824,28 +6825,61 @@ def split_merge_docs(docs_with_score, tokenizer=None, max_input_tokens=None, doc return docs_with_score, 0 elif docs_token_handling in [None, 'split_or_merge']: assert tokenizer - tokens_before_split = [get_token_count(x + joiner, tokenizer) for x in - [x[0].page_content for x in docs_with_score]] + # see if need to split + # account for joiner tokens + joiner_tokens = get_token_count(joiner, tokenizer) + doc_chunk_size = max(64, min(max_input_tokens, + max(64, max_input_tokens - joiner_tokens * len(docs_with_score)))) + + if do_first_semantic_split and hf_embedding_model is not None and 'model' in hf_embedding_model: + # https://python.langchain.com/v0.1/docs/modules/data_connection/document_transformers/semantic-chunker/ + from langchain_experimental.text_splitter import SemanticChunker + text_splitter0 = SemanticChunker(hf_embedding_model['model']) + else: + text_splitter0 = None + # skip split if not necessary, since expensive for some reason - do_split &= any([x > max_input_tokens for x in tokens_before_split]) - if do_split: + text_splitter1 = H2OCharacterTextSplitter.from_huggingface_tokenizer( + tokenizer, chunk_size=doc_chunk_size, chunk_overlap=0, + separators=[". "], + ) + text_splitter2 = H2OCharacterTextSplitter.from_huggingface_tokenizer( + tokenizer, chunk_size=doc_chunk_size, chunk_overlap=0, + ) + # https://python.langchain.com/v0.1/docs/modules/data_connection/document_transformers/recursive_text_splitter/ + text_splitter3 = H2OCharacterTextSplitter.from_huggingface_tokenizer( + tokenizer, chunk_size=doc_chunk_size, chunk_overlap=0, + separators=[ + "\n\n", + "\n", + " ", + ".", + ",", + "\u200b", # Zero-width space + "\uff0c", # Fullwidth comma + "\u3001", # Ideographic comma + "\uff0e", # Fullwidth full stop + "\u3002", # Ideographic full stop + "", + ], + ) + text_splitters = dict(semantic=text_splitter0, sentence=text_splitter1, normal=text_splitter2, + multilingual=text_splitter3) + text_splitters = {k: v for k, v in text_splitters.items() if v is not None} + + did_split = False + for splitter_type, text_splitter in text_splitters.items(): + tokens_before_split = [get_token_count(x + joiner, tokenizer) for x in + [x[0].page_content for x in docs_with_score]] - if do_first_semantic_split and hf_embedding_model is not None and 'model' in hf_embedding_model: - from langchain_experimental.text_splitter import SemanticChunker - text_splitter = SemanticChunker(hf_embedding_model['model']) - docs_with_score = text_splitter.create_documents(docs_with_score) + do_split &= any([x > max_input_tokens for x in tokens_before_split]) + if not do_split: + break + did_split = True if verbose: print('tokens_before_split=%s' % tokens_before_split, flush=True) - # see if need to split - # account for joiner tokens - joiner_tokens = get_token_count(joiner, tokenizer) - doc_chunk_size = max(64, min(max_input_tokens, - max(64, max_input_tokens - joiner_tokens * len(docs_with_score)))) - text_splitter = H2OCharacterTextSplitter.from_huggingface_tokenizer( - tokenizer, chunk_size=doc_chunk_size, chunk_overlap=0 - ) [x[0].metadata.update(dict(docscore=x[1], doci=doci, ntokens=tokens_before_split[doci])) for doci, x in enumerate(docs_with_score)] docs = [x[0] for x in docs_with_score] @@ -6859,11 +6893,17 @@ def split_merge_docs(docs_with_score, tokenizer=None, max_input_tokens=None, doc docs_new = [x for _, x in sorted(zip(doci_new, docs_new), key=lambda pair: pair[0])] docs_with_score = [(x, x.metadata['docscore']) for x in docs_new] - tokens_after_split = [get_token_count(x + joiner, tokenizer) for x in - [x[0].page_content for x in docs_with_score]] if verbose: + tokens_after_split = [get_token_count(x + joiner, tokenizer) for x in + [x[0].page_content for x in docs_with_score]] print('tokens_after_split=%s' % tokens_after_split, flush=True) + if splitter_type == 'sentence' and len(docs_with_score) > 1: + # puts '. ' on next end of chunk, re-attach to end of previous chunk + docs_with_score = [ + (Document(x[0].page_content[2 if xi > 0 else 0:] + '.', metadata=x[0].metadata), x[1]) for xi, x in + enumerate(docs_with_score)] + docs_with_score_new = [] k = 0 while k < len(docs_with_score): @@ -6880,7 +6920,7 @@ def split_merge_docs(docs_with_score, tokenizer=None, max_input_tokens=None, doc doc1 = Document(page_content=new_page_content, metadata=new_metadata) docs_with_score_new.append((doc1, new_score)) - if do_split: + if did_split: assert one_doc_size is None, "Split failed: %s" % one_doc_size elif one_doc_size is not None: # chopped @@ -8021,6 +8061,12 @@ def get_chain(query=None, ) # group docs if desired/can to fill context to avoid multiple LLM calls or too large chunks + # only do first semantic split if have GPU + if 'model' in hf_embedding_model and not use_openai_embedding and hasattr(hf_embedding_model['model'], + 'model_kwargs'): + do_first_semantic_split = hf_embedding_model['model'].model_kwargs.get('device') not in ['cpu'] + else: + do_first_semantic_split = False docs_with_score, max_doc_tokens = split_merge_docs(docs_with_score, tokenizer, max_input_tokens=max_input_tokens, @@ -8028,6 +8074,7 @@ def get_chain(query=None, joiner=docs_joiner if not doing_grounding else "Document xx", non_doc_prompt=estimated_full_prompt, hf_embedding_model=hf_embedding_model, + do_first_semantic_split=do_first_semantic_split, verbose=verbose) # in case docs_with_score grew due to splitting, limit again by top_k_docs if top_k_docs > 0: diff --git a/src/version.py b/src/version.py index f2d4dfb07..8784a104b 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1 @@ -__version__ = "1ad4c7e4622c4112c323bee41183162933927342" +__version__ = "67c8a57a45f9ca7fbb2127bf955f8b5a82b64efa" From 9aa4ab14603c51481f4964bc311db2f7a4e19f23 Mon Sep 17 00:00:00 2001 From: "Jonathan C. McKinney" Date: Fri, 10 May 2024 23:53:58 -0700 Subject: [PATCH 3/9] Update test --- src/version.py | 2 +- tests/test_langchain_units.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/version.py b/src/version.py index 8784a104b..01555e3dc 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1 @@ -__version__ = "67c8a57a45f9ca7fbb2127bf955f8b5a82b64efa" +__version__ = "e65cd157863e45ef71516e62d064185d4ccade2a" diff --git a/tests/test_langchain_units.py b/tests/test_langchain_units.py index 81b1be9ca..dffcb39fa 100644 --- a/tests/test_langchain_units.py +++ b/tests/test_langchain_units.py @@ -8,16 +8,15 @@ import pytest -from src.gen import get_model_retry from tests.test_client_calls import texts_helium1, texts_helium2, texts_helium3, texts_helium4, texts_helium5, \ texts_simple, texts_long from tests.utils import wrap_test_forked, kill_weaviate, make_user_path_test from src.enums import DocumentSubset, LangChainAction, LangChainMode, LangChainTypes, DocumentChoice, \ docs_joiner_default, docs_token_handling_default, db_types, db_types_full -from src.gpt_langchain import get_persist_directory, get_db, get_documents, length_db1, _run_qa_db, split_merge_docs, \ - get_hyde_acc from src.utils import zip_data, download_simple, get_ngpus_vis, get_mem_gpus, have_faiss, remove, get_kwargs, \ FakeTokenizer, get_token_count, flatten_list, tar_data +from src.gpt_langchain import get_persist_directory, get_db, get_documents, length_db1, _run_qa_db, split_merge_docs, \ + get_hyde_acc have_openai_key = os.environ.get('OPENAI_API_KEY') is not None have_replicate_key = os.environ.get('REPLICATE_API_TOKEN') is not None @@ -302,6 +301,7 @@ def get_test_model(base_model='h2oai/h2ogpt-oig-oasst1-512-6_9b', force_t5_type=False, verbose=False) + from src.gen import get_model_retry model, tokenizer, device = get_model_retry(reward_type=False, **get_kwargs(get_model, exclude_names=['reward_type'], **all_kwargs)) return model, tokenizer, base_model, prompt_type @@ -2059,7 +2059,7 @@ def test_merge_docs(data_kind, max_input_tokens): assert len(docs_with_score_new) == 3 if max_input_tokens == 1024 else 1 assert all([x <= max_input_tokens for x in tokens]) elif data_kind == 'long': - assert len(docs_with_score_new) == 6 if max_input_tokens == 1024 else 6 + assert len(docs_with_score_new) == 41 if max_input_tokens == 1024 else 6, len(docs_with_score_new) assert all([x <= max_input_tokens for x in tokens]) @@ -2076,6 +2076,8 @@ def test_split_and_merge(): tokenizer, **kwargs) assert len(docs_with_score) == 6 + # ensure docuemnt doesn't start with . from sentence splitting + assert docs_with_score[0][0].page_content.startswith('Y') @wrap_test_forked From 3406e93e91fc8dd8b0a39e24c7e7104fad1d7dc5 Mon Sep 17 00:00:00 2001 From: "Jonathan C. McKinney" Date: Sat, 11 May 2024 00:39:46 -0700 Subject: [PATCH 4/9] Semantic or sentence splitting for normal chunking too --- src/gpt_langchain.py | 206 +++++++-------------------------------- src/make_db.py | 12 ++- src/utils_langchain.py | 212 +++++++++++++++++++++++++++++++++++++++-- src/version.py | 2 +- 4 files changed, 250 insertions(+), 182 deletions(-) diff --git a/src/gpt_langchain.py b/src/gpt_langchain.py index 99d4dd13a..14dd04590 100644 --- a/src/gpt_langchain.py +++ b/src/gpt_langchain.py @@ -85,7 +85,8 @@ is_vision_model, is_gradio_vision_model, is_json_model from src.serpapi import H2OSerpAPIWrapper from utils_langchain import StreamingGradioCallbackHandler, _chunk_sources, _add_meta, add_parser, fix_json_meta, \ - load_general_summarization_chain, H2OHuggingFaceHubEmbeddings, make_sources_file + load_general_summarization_chain, H2OHuggingFaceHubEmbeddings, make_sources_file, select_docs_with_score, \ + split_merge_docs # to check imports # find ./src -name '*.py' | xargs awk '{ if (sub(/\\$/, "")) printf "%s ", $0; else print; }' | grep 'from langchain\.' | sed 's/^[ \t]*//' > go.py @@ -3613,6 +3614,9 @@ def file_to_doc(file, is_public=False, from_ui=True, + + hf_embedding_model=None, + use_openai_embedding=False, ): # SOME AUTODETECTION LOGIC FOR URL VS TEXT @@ -3669,7 +3673,9 @@ def file_to_doc(file, set_audio_types1 = set_audio_types assert db_type is not None - chunk_sources = functools.partial(_chunk_sources, chunk=chunk, chunk_size=chunk_size, db_type=db_type) + chunk_sources = functools.partial(_chunk_sources, chunk=chunk, chunk_size=chunk_size, db_type=db_type, + hf_embedding_model=hf_embedding_model, use_openai_embedding=use_openai_embedding, + verbose=verbose) add_meta = functools.partial(_add_meta, headsize=headsize, filei=filei) # FIXME: if zip, file index order will not be correct if other files involved path_to_docs_func = functools.partial(path_to_docs, @@ -3723,6 +3729,9 @@ def file_to_doc(file, is_public=is_public, from_ui=from_ui, + + hf_embedding_model=hf_embedding_model, + use_openai_embedding=use_openai_embedding, ) if file is None: @@ -4619,6 +4628,9 @@ def path_to_doc1(file, is_public=False, from_ui=True, + + hf_embedding_model=None, + use_openai_embedding=False, ): assert db_type is not None if verbose: @@ -4681,6 +4693,9 @@ def path_to_doc1(file, selected_file_types=selected_file_types, is_public=is_public, from_ui=from_ui, + + hf_embedding_model=hf_embedding_model, + use_openai_embedding=use_openai_embedding, ) except BaseException as e: print("Failed to ingest %s due to %s" % (file, traceback.format_exc())) @@ -4764,6 +4779,9 @@ def path_to_docs(path_or_paths, selected_file_types=None, from_ui=True, + + use_openai_embedding=False, + hf_embedding_model=None, ): if verbose: print("BEGIN Consuming path_or_paths=%s url=%s text=%s" % (path_or_paths, url, text), flush=True) @@ -4906,6 +4924,9 @@ def path_to_docs(path_or_paths, is_public=is_public, from_ui=from_ui, + + hf_embedding_model=hf_embedding_model, + use_openai_embedding=use_openai_embedding, ) if is_public: @@ -4923,6 +4944,7 @@ def no_tqdm(x): filei0 = filei if n_jobs != 1 and len(globs_non_image_types) > 1: + kwargs['hf_embedding_model'] = None # can't fork and use CUDA # avoid nesting, e.g. upload 1 zip and then inside many files # harder to handle if upload many zips with many files, inner parallel one will be disabled by joblib documents = ProgressParallel(n_jobs=n_jobs, verbose=10 if verbose else 0, backend='multiprocessing')( @@ -5520,7 +5542,9 @@ def _make_db(use_openai_embedding=False, sources = [] if not db: - chunk_sources = functools.partial(_chunk_sources, chunk=chunk, chunk_size=chunk_size, db_type=db_type) + chunk_sources = functools.partial(_chunk_sources, chunk=chunk, chunk_size=chunk_size, db_type=db_type, + hf_embedding_model=hf_embedding_model, + use_openai_embedding=use_openai_embedding, verbose=verbose) if langchain_mode in ['wiki_full']: from read_wiki_full import get_all_documents small_test = None @@ -5602,6 +5626,9 @@ def _make_db(use_openai_embedding=False, is_public=False, from_ui=True, + + hf_embedding_model=hf_embedding_model, + use_openai_embedding=use_openai_embedding, ) new_metadata_sources = set([x.metadata['source'] for x in sources1]) if new_metadata_sources: @@ -6777,168 +6804,6 @@ def _get_docs_with_score(query, k_db, return docs_with_score -def select_docs_with_score(docs_with_score, top_k_docs, one_doc_size): - if top_k_docs > 0: - docs_with_score = docs_with_score[:top_k_docs] - elif one_doc_size is not None: - docs_with_score = [(docs_with_score[0][:one_doc_size], docs_with_score[0][1])] - else: - # do nothing - pass - return docs_with_score - - -class H2OCharacterTextSplitter(RecursiveCharacterTextSplitter): - def __init__( - self, - separators: Optional[List[str]] = None, - keep_separator: bool = True, - is_separator_regex: bool = False, - **kwargs: Any, - ) -> None: - """Create a new TextSplitter.""" - super().__init__(separators=separators, keep_separator=keep_separator, is_separator_regex=is_separator_regex, - **kwargs) - self._separators = separators or ["\n\n", "\n", " ", " ", ""] - - @classmethod - def from_huggingface_tokenizer(cls, tokenizer: Any, **kwargs: Any) -> TextSplitter: - def _huggingface_tokenizer_length(text: str) -> int: - return get_token_count(text, tokenizer, add_special_tokens=False) - - return cls(length_function=_huggingface_tokenizer_length, **kwargs) - - -def split_merge_docs(docs_with_score, tokenizer=None, max_input_tokens=None, docs_token_handling=None, - joiner=docs_joiner_default, - non_doc_prompt='', - do_split=True, - do_first_semantic_split=False, - hf_embedding_model=None, - verbose=False): - # NOTE: Could use joiner=\n\n, but if PDF and continues, might want just full continue with joiner='' - # NOTE: assume max_input_tokens already processed if was -1 and accounts for model_max_len and is per-llm call - if max_input_tokens is not None: - max_input_tokens -= get_token_count(non_doc_prompt, tokenizer) - - if docs_token_handling in ['chunk']: - return docs_with_score, 0 - elif docs_token_handling in [None, 'split_or_merge']: - assert tokenizer - # see if need to split - # account for joiner tokens - joiner_tokens = get_token_count(joiner, tokenizer) - doc_chunk_size = max(64, min(max_input_tokens, - max(64, max_input_tokens - joiner_tokens * len(docs_with_score)))) - - if do_first_semantic_split and hf_embedding_model is not None and 'model' in hf_embedding_model: - # https://python.langchain.com/v0.1/docs/modules/data_connection/document_transformers/semantic-chunker/ - from langchain_experimental.text_splitter import SemanticChunker - text_splitter0 = SemanticChunker(hf_embedding_model['model']) - else: - text_splitter0 = None - - # skip split if not necessary, since expensive for some reason - text_splitter1 = H2OCharacterTextSplitter.from_huggingface_tokenizer( - tokenizer, chunk_size=doc_chunk_size, chunk_overlap=0, - separators=[". "], - ) - text_splitter2 = H2OCharacterTextSplitter.from_huggingface_tokenizer( - tokenizer, chunk_size=doc_chunk_size, chunk_overlap=0, - ) - # https://python.langchain.com/v0.1/docs/modules/data_connection/document_transformers/recursive_text_splitter/ - text_splitter3 = H2OCharacterTextSplitter.from_huggingface_tokenizer( - tokenizer, chunk_size=doc_chunk_size, chunk_overlap=0, - separators=[ - "\n\n", - "\n", - " ", - ".", - ",", - "\u200b", # Zero-width space - "\uff0c", # Fullwidth comma - "\u3001", # Ideographic comma - "\uff0e", # Fullwidth full stop - "\u3002", # Ideographic full stop - "", - ], - ) - text_splitters = dict(semantic=text_splitter0, sentence=text_splitter1, normal=text_splitter2, - multilingual=text_splitter3) - text_splitters = {k: v for k, v in text_splitters.items() if v is not None} - - did_split = False - for splitter_type, text_splitter in text_splitters.items(): - tokens_before_split = [get_token_count(x + joiner, tokenizer) for x in - [x[0].page_content for x in docs_with_score]] - - do_split &= any([x > max_input_tokens for x in tokens_before_split]) - if not do_split: - break - did_split = True - - if verbose: - print('tokens_before_split=%s' % tokens_before_split, flush=True) - - [x[0].metadata.update(dict(docscore=x[1], doci=doci, ntokens=tokens_before_split[doci])) for doci, x in - enumerate(docs_with_score)] - docs = [x[0] for x in docs_with_score] - # only split those that need to be split, else recursive splitter goes too nuts and takes too long - docs_to_split = [x for x in docs if x.metadata['ntokens'] > doc_chunk_size] - docs_to_not_split = [x for x in docs if x.metadata['ntokens'] <= doc_chunk_size] - docs_split_new = flatten_list([text_splitter.split_documents([x]) for x in docs_to_split]) - docs_new = docs_to_not_split + docs_split_new - doci_new = [x.metadata['doci'] for x in docs_new] - # order back by doci - docs_new = [x for _, x in sorted(zip(doci_new, docs_new), key=lambda pair: pair[0])] - docs_with_score = [(x, x.metadata['docscore']) for x in docs_new] - - if verbose: - tokens_after_split = [get_token_count(x + joiner, tokenizer) for x in - [x[0].page_content for x in docs_with_score]] - print('tokens_after_split=%s' % tokens_after_split, flush=True) - - if splitter_type == 'sentence' and len(docs_with_score) > 1: - # puts '. ' on next end of chunk, re-attach to end of previous chunk - docs_with_score = [ - (Document(x[0].page_content[2 if xi > 0 else 0:] + '.', metadata=x[0].metadata), x[1]) for xi, x in - enumerate(docs_with_score)] - - docs_with_score_new = [] - k = 0 - while k < len(docs_with_score): - # means use max_input_tokens to ensure model gets no more than max_input_tokens each map - top_k_docs, one_doc_size, num_doc_tokens = \ - get_docs_tokens(tokenizer, - text_context_list=[x[0].page_content for x in docs_with_score[k:]], - max_input_tokens=max_input_tokens) - docs_with_score1 = select_docs_with_score(docs_with_score[k:], top_k_docs, one_doc_size) - new_score = docs_with_score1[0][1] - new_page_content = joiner.join([x[0].page_content for x in docs_with_score1]) - new_metadata = docs_with_score1[0][0].metadata.copy() - new_metadata['source'] = joiner.join(set([x[0].metadata['source'] for x in docs_with_score1])) - doc1 = Document(page_content=new_page_content, metadata=new_metadata) - docs_with_score_new.append((doc1, new_score)) - - if did_split: - assert one_doc_size is None, "Split failed: %s" % one_doc_size - elif one_doc_size is not None: - # chopped - assert top_k_docs == 1 - assert top_k_docs >= 1 - k += top_k_docs - - tokens_after_merge = [get_token_count(x + joiner, tokenizer) for x in - [x[0].page_content for x in docs_with_score_new]] - if verbose: - print('tokens_after_merge=%s' % tokens_after_merge, flush=True) - - max_tokens_after_merge = max(tokens_after_merge) if tokens_after_merge else 0 - return docs_with_score_new, max_tokens_after_merge - else: - raise ValueError("No such docs_token_handling=%s" % docs_token_handling) - - def get_single_document(document_choice, db, extension=None): if isinstance(document_choice, str): document_choice = [document_choice] @@ -8060,13 +7925,6 @@ def get_chain(query=None, # nothing, just getting base amount for each call ) - # group docs if desired/can to fill context to avoid multiple LLM calls or too large chunks - # only do first semantic split if have GPU - if 'model' in hf_embedding_model and not use_openai_embedding and hasattr(hf_embedding_model['model'], - 'model_kwargs'): - do_first_semantic_split = hf_embedding_model['model'].model_kwargs.get('device') not in ['cpu'] - else: - do_first_semantic_split = False docs_with_score, max_doc_tokens = split_merge_docs(docs_with_score, tokenizer, max_input_tokens=max_input_tokens, @@ -8074,7 +7932,6 @@ def get_chain(query=None, joiner=docs_joiner if not doing_grounding else "Document xx", non_doc_prompt=estimated_full_prompt, hf_embedding_model=hf_embedding_model, - do_first_semantic_split=do_first_semantic_split, verbose=verbose) # in case docs_with_score grew due to splitting, limit again by top_k_docs if top_k_docs > 0: @@ -9016,6 +8873,9 @@ def _update_user_db(file, is_public=is_public, from_ui=from_ui, + + use_openai_embedding=use_openai_embedding, + hf_embedding_model=hf_embedding_model, ) exceptions = [x for x in sources if x.metadata.get('exception')] exceptions_strs = [x.metadata['exception'] for x in exceptions] diff --git a/src/make_db.py b/src/make_db.py index 65e6ad8b1..e5a3bb0a0 100644 --- a/src/make_db.py +++ b/src/make_db.py @@ -51,7 +51,11 @@ def glob_to_db(user_path, chunk=True, chunk_size=512, verbose=False, db_type=None, selected_file_types=None, - is_public=False): + is_public=False, + + hf_embedding_model=None, + use_openai_embedding=False, + ): assert db_type is not None loaders_and_settings = dict( @@ -100,6 +104,9 @@ def glob_to_db(user_path, chunk=True, chunk_size=512, verbose=False, db_type=db_type, is_public=is_public, + + hf_embedding_model=hf_embedding_model, + use_openai_embedding=use_openai_embedding, ) sources1 = path_to_docs(user_path, url=url, @@ -371,6 +378,9 @@ def make_db_main(use_openai_embedding: bool = False, selected_file_types=selected_file_types, is_public=False, + + hf_embedding_model=hf_embedding_model, + use_openai_embedding=use_openai_embedding, ) exceptions = [x for x in sources if x.metadata.get('exception')] print("Exceptions: %s/%s %s" % (len(exceptions), len(sources), exceptions), flush=True) diff --git a/src/utils_langchain.py b/src/utils_langchain.py index f25186f49..f1089e6fb 100644 --- a/src/utils_langchain.py +++ b/src/utils_langchain.py @@ -18,8 +18,11 @@ from langchain.chains.summarize.chain import _load_stuff_chain, _load_refine_chain, _load_map_reduce_chain from langchain.schema.language_model import BaseLanguageModel from langchain_community.embeddings import HuggingFaceHubEmbeddings +from langchain_text_splitters import TextSplitter -from src.utils import hash_file, get_sha, split_list, makedirs +from src.enums import docs_joiner_default +from src.utils import hash_file, get_sha, split_list, makedirs, flatten_list, get_token_count, get_docs_tokens, \ + FakeTokenizer from langchain.callbacks.base import BaseCallbackHandler, Callbacks from langchain.schema import LLMResult @@ -58,7 +61,7 @@ def on_llm_start( def on_llm_new_token(self, token: str, **kwargs: Any) -> None: """Run on new LLM token. Only available when streaming is enabled.""" if False and \ - self.tgen0 is not None and self.max_time is not None and (time.time() - self.tgen0) > self.max_time: + self.tgen0 is not None and self.max_time is not None and (time.time() - self.tgen0) > self.max_time: if self.verbose: print("Took too long in StreamingGradioCallbackHandler: %s" % (time.time() - self.tgen0), flush=True) self.text_queue.put(self.stop_signal) @@ -100,7 +103,180 @@ def __next__(self): return value -def _chunk_sources(sources, chunk=True, chunk_size=512, language=None, db_type=None): +class H2OCharacterTextSplitter(RecursiveCharacterTextSplitter): + def __init__( + self, + separators: Optional[List[str]] = None, + keep_separator: bool = True, + is_separator_regex: bool = False, + **kwargs: Any, + ) -> None: + """Create a new TextSplitter.""" + super().__init__(separators=separators, keep_separator=keep_separator, is_separator_regex=is_separator_regex, + **kwargs) + self._separators = separators or ["\n\n", "\n", " ", " ", ""] + + @classmethod + def from_huggingface_tokenizer(cls, tokenizer: Any, **kwargs: Any) -> TextSplitter: + def _huggingface_tokenizer_length(text: str) -> int: + return get_token_count(text, tokenizer, add_special_tokens=False) + + return cls(length_function=_huggingface_tokenizer_length, **kwargs) + + +def select_docs_with_score(docs_with_score, top_k_docs, one_doc_size): + if top_k_docs > 0: + docs_with_score = docs_with_score[:top_k_docs] + elif one_doc_size is not None: + docs_with_score = [(docs_with_score[0][:one_doc_size], docs_with_score[0][1])] + else: + # do nothing + pass + return docs_with_score + + +def split_merge_docs(docs_with_score, tokenizer=None, max_input_tokens=None, docs_token_handling=None, + joiner=docs_joiner_default, + non_doc_prompt='', + do_split=True, + hf_embedding_model=None, + use_openai_embedding=False, + verbose=False): + # group docs if desired/can to fill context to avoid multiple LLM calls or too large chunks + # only do first semantic split if have GPU + if hf_embedding_model and \ + 'model' in hf_embedding_model and \ + not use_openai_embedding and \ + hasattr(hf_embedding_model['model'], 'model_kwargs'): + do_first_semantic_split = hf_embedding_model['model'].model_kwargs.get('device') not in ['cpu'] + else: + do_first_semantic_split = False + + # NOTE: Could use joiner=\n\n, but if PDF and continues, might want just full continue with joiner='' + # NOTE: assume max_input_tokens already processed if was -1 and accounts for model_max_len and is per-llm call + if max_input_tokens is not None: + max_input_tokens -= get_token_count(non_doc_prompt, tokenizer) + + if docs_token_handling in ['chunk']: + return docs_with_score, 0 + elif docs_token_handling in [None, 'split_or_merge']: + assert tokenizer + # see if need to split + # account for joiner tokens + joiner_tokens = get_token_count(joiner, tokenizer) + doc_chunk_size = max(64, min(max_input_tokens, + max(64, max_input_tokens - joiner_tokens * len(docs_with_score)))) + + if do_first_semantic_split and hf_embedding_model is not None and 'model' in hf_embedding_model: + # https://python.langchain.com/v0.1/docs/modules/data_connection/document_transformers/semantic-chunker/ + from langchain_experimental.text_splitter import SemanticChunker + text_splitter0 = SemanticChunker(hf_embedding_model['model']) + else: + text_splitter0 = None + + # skip split if not necessary, since expensive for some reason + text_splitter1 = H2OCharacterTextSplitter.from_huggingface_tokenizer( + tokenizer, chunk_size=doc_chunk_size, chunk_overlap=0, + separators=[". "], + ) + text_splitter2 = H2OCharacterTextSplitter.from_huggingface_tokenizer( + tokenizer, chunk_size=doc_chunk_size, chunk_overlap=0, + ) + # https://python.langchain.com/v0.1/docs/modules/data_connection/document_transformers/recursive_text_splitter/ + text_splitter3 = H2OCharacterTextSplitter.from_huggingface_tokenizer( + tokenizer, chunk_size=doc_chunk_size, chunk_overlap=0, + separators=[ + "\n\n", + "\n", + " ", + ".", + ",", + "\u200b", # Zero-width space + "\uff0c", # Fullwidth comma + "\u3001", # Ideographic comma + "\uff0e", # Fullwidth full stop + "\u3002", # Ideographic full stop + "", + ], + ) + text_splitters = dict(semantic=text_splitter0, sentence=text_splitter1, normal=text_splitter2, + multilingual=text_splitter3) + text_splitters = {k: v for k, v in text_splitters.items() if v is not None} + + did_split = False + for splitter_type, text_splitter in text_splitters.items(): + tokens_before_split = [get_token_count(x + joiner, tokenizer) for x in + [x[0].page_content for x in docs_with_score]] + + do_split &= any([x > max_input_tokens for x in tokens_before_split]) + if not do_split: + break + did_split = True + + if verbose: + print('tokens_before_split=%s' % tokens_before_split, flush=True) + + [x[0].metadata.update(dict(docscore=x[1], doci=doci, ntokens=tokens_before_split[doci])) for doci, x in + enumerate(docs_with_score)] + docs = [x[0] for x in docs_with_score] + # only split those that need to be split, else recursive splitter goes too nuts and takes too long + docs_to_split = [x for x in docs if x.metadata['ntokens'] > doc_chunk_size] + docs_to_not_split = [x for x in docs if x.metadata['ntokens'] <= doc_chunk_size] + docs_split_new = flatten_list([text_splitter.split_documents([x]) for x in docs_to_split]) + docs_new = docs_to_not_split + docs_split_new + doci_new = [x.metadata['doci'] for x in docs_new] + # order back by doci + docs_new = [x for _, x in sorted(zip(doci_new, docs_new), key=lambda pair: pair[0])] + docs_with_score = [(x, x.metadata['docscore']) for x in docs_new] + + if verbose: + tokens_after_split = [get_token_count(x + joiner, tokenizer) for x in + [x[0].page_content for x in docs_with_score]] + print('tokens_after_split=%s' % tokens_after_split, flush=True) + + if splitter_type == 'sentence' and len(docs_with_score) > 1: + # puts '. ' on next end of chunk, re-attach to end of previous chunk + docs_with_score = [ + (Document(x[0].page_content[2 if xi > 0 else 0:] + '.', metadata=x[0].metadata), x[1]) for xi, x in + enumerate(docs_with_score)] + + docs_with_score_new = [] + k = 0 + while k < len(docs_with_score): + # means use max_input_tokens to ensure model gets no more than max_input_tokens each map + top_k_docs, one_doc_size, num_doc_tokens = \ + get_docs_tokens(tokenizer, + text_context_list=[x[0].page_content for x in docs_with_score[k:]], + max_input_tokens=max_input_tokens) + docs_with_score1 = select_docs_with_score(docs_with_score[k:], top_k_docs, one_doc_size) + new_score = docs_with_score1[0][1] + new_page_content = joiner.join([x[0].page_content for x in docs_with_score1]) + new_metadata = docs_with_score1[0][0].metadata.copy() + new_metadata['source'] = joiner.join(set([x[0].metadata['source'] for x in docs_with_score1])) + doc1 = Document(page_content=new_page_content, metadata=new_metadata) + docs_with_score_new.append((doc1, new_score)) + + if did_split: + assert one_doc_size is None, "Split failed: %s" % one_doc_size + elif one_doc_size is not None: + # chopped + assert top_k_docs == 1 + assert top_k_docs >= 1 + k += top_k_docs + + tokens_after_merge = [get_token_count(x + joiner, tokenizer) for x in + [x[0].page_content for x in docs_with_score_new]] + if verbose: + print('tokens_after_merge=%s' % tokens_after_merge, flush=True) + + max_tokens_after_merge = max(tokens_after_merge) if tokens_after_merge else 0 + return docs_with_score_new, max_tokens_after_merge + else: + raise ValueError("No such docs_token_handling=%s" % docs_token_handling) + + +def _chunk_sources(sources, chunk=True, chunk_size=512, language=None, db_type=None, + new_splitter=True, hf_embedding_model=None, use_openai_embedding=False, verbose=False): assert db_type is not None if not isinstance(sources, (list, tuple, types.GeneratorType)) and not callable(sources): @@ -125,9 +301,30 @@ def _chunk_sources(sources, chunk=True, chunk_size=512, language=None, db_type=N else: separators = ["\n\n", "\n", " ", ""] keep_separator = False - splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=0, keep_separator=keep_separator, - separators=separators) - source_chunks = splitter.split_documents(sources) + if not new_splitter: + splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=0, + keep_separator=keep_separator, + separators=separators) + source_chunks = splitter.split_documents(sources) + else: + try: + tokenizer = FakeTokenizer(model_max_length=max(20, chunk_size - 50), is_openai=True) + sources_with_score = [(x, 1) for x in sources] + source_chunks_with_score, max_tokens_after_merge = \ + split_merge_docs(sources_with_score, tokenizer=tokenizer, + max_input_tokens=chunk_size, non_doc_prompt='', + do_split=True, + hf_embedding_model=hf_embedding_model if not use_openai_embedding else None, + verbose=verbose) + source_chunks = [x[0] for x in source_chunks_with_score] + except BaseException: + if os.getenv('HARD_ASSERTS'): + raise + print("Failed to split with new method, use old method: %s" % str(e)) + splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=0, + keep_separator=keep_separator, + separators=separators) + source_chunks = splitter.split_documents(sources) # currently in order, but when pull from db won't be, so mark order and document by hash [x.metadata.update(dict(chunk_id=chunk_id)) for chunk_id, x in enumerate(source_chunks)] @@ -154,7 +351,7 @@ def _add_meta(docs1, file, headsize=50, filei=0, parser='NotSet', file_as_source file_extension = pathlib.Path(file).suffix hashid = hash_file(file) else: - file_extension = str(file) # not file, just show full thing + file_extension = str(type(file)) hashid = get_sha(file) doc_hash = str(uuid.uuid4())[:10] if not isinstance(docs1, (list, tuple, types.GeneratorType)): @@ -183,6 +380,7 @@ def fix_json_meta(docs1): class H2OMapReduceDocumentsChain(MapReduceDocumentsChain): allow_map_1 = True which = 'map' + def combine_docs( self, docs: List[Document], diff --git a/src/version.py b/src/version.py index 01555e3dc..24fa3c8b4 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1 @@ -__version__ = "e65cd157863e45ef71516e62d064185d4ccade2a" +__version__ = "9aa4ab14603c51481f4964bc311db2f7a4e19f23" From 8d2af1deb95aaea6eb3b9adf9523790be52d7f41 Mon Sep 17 00:00:00 2001 From: "Jonathan C. McKinney" Date: Sat, 11 May 2024 01:02:38 -0700 Subject: [PATCH 5/9] Break pure text in doc view by sentences --- src/gradio_runner.py | 3 ++- src/utils.py | 10 ++++++++-- src/version.py | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/gradio_runner.py b/src/gradio_runner.py index bc31e38ed..e082c36e1 100644 --- a/src/gradio_runner.py +++ b/src/gradio_runner.py @@ -7018,7 +7018,8 @@ def show_doc(db1s, selection_docs_state1, requests_state1, try: with open(file, 'rt') as f: content = f.read() - content = f"```text\n{content}\n```" + #content = f"```text\n{content}\n```" + content = text_to_html(content, api=api) return dummy1, dummy1, dummy1, gr.update(visible=True, value=content), dummy1, dummy1, dummy1, dummy1 except: return dummy_ret diff --git a/src/utils.py b/src/utils.py index 89b16d663..b4ad2c0a6 100644 --- a/src/utils.py +++ b/src/utils.py @@ -1698,6 +1698,12 @@ def dict_to_html(x, small=True, api=False): return res +def split_into_sentences(text): + # Split text by specified punctuation followed by space or end of text + sentences = re.split(r'(?<=[.!?]) +', text) + return sentences + + def text_to_html(x, api=False): if api: return x @@ -1711,11 +1717,11 @@ def text_to_html(x, api=False): white-space: -o-pre-wrap; word-wrap: break-word; } - + 
 %s
-""" % x +""" % '
'.join(split_into_sentences(x)) def lg_to_gr( diff --git a/src/version.py b/src/version.py index 24fa3c8b4..d0bae64fc 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1 @@ -__version__ = "9aa4ab14603c51481f4964bc311db2f7a4e19f23" +__version__ = "3406e93e91fc8dd8b0a39e24c7e7104fad1d7dc5" From ad975db77c5f6387a43207935dedb65d3db60fd6 Mon Sep 17 00:00:00 2001 From: "Jonathan C. McKinney" Date: Sat, 11 May 2024 01:16:08 -0700 Subject: [PATCH 6/9] Fix top_k_docs filtering when had one large block at first --- src/gpt_langchain.py | 14 +++++++------- src/version.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/gpt_langchain.py b/src/gpt_langchain.py index 14dd04590..e003aa9bd 100644 --- a/src/gpt_langchain.py +++ b/src/gpt_langchain.py @@ -7720,8 +7720,6 @@ def get_chain(query=None, text_context_list=text_context_list, chunk_id_filter=chunk_id_filter) - if top_k_docs == -1: - top_k_docs = len(db_documents) # similar to langchain's chroma's _results_to_docs_and_scores docs_with_score = [(Document(page_content=result[0], metadata=result[1] or {}), 0) for result in zip(db_documents, db_metadatas)] @@ -7754,7 +7752,8 @@ def get_chain(query=None, ] docs_with_score = docs_with_score2 - docs_with_score = docs_with_score[:top_k_docs] + top_k_docs_sample = len(db_documents) if top_k_docs == -1 else top_k_docs + docs_with_score = docs_with_score[:top_k_docs_sample] docs = [x[0] for x in docs_with_score] scores = [x[1] for x in docs_with_score] else: @@ -7883,15 +7882,16 @@ def get_chain(query=None, get_llm(**llm_kwargs) # avoid craziness + top_k_docs_sample = len(docs_with_score) if top_k_docs == -1 else top_k_docs if 0 < top_k_docs_trial < max_chunks: # avoid craziness if top_k_docs == -1: - top_k_docs = top_k_docs_trial + top_k_docs_sample = top_k_docs_trial else: - top_k_docs = min(top_k_docs, top_k_docs_trial) + top_k_docs_sample = min(top_k_docs, top_k_docs_trial) elif top_k_docs_trial >= max_chunks: - top_k_docs = max_chunks - docs_with_score = select_docs_with_score(docs_with_score, top_k_docs, one_doc_size) + top_k_docs_sample = max_chunks + docs_with_score = select_docs_with_score(docs_with_score, top_k_docs_sample, one_doc_size) else: # don't reduce, except listen to top_k_docs and max_total_input_tokens one_doc_size = None diff --git a/src/version.py b/src/version.py index d0bae64fc..be9662c71 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1 @@ -__version__ = "3406e93e91fc8dd8b0a39e24c7e7104fad1d7dc5" +__version__ = "8d2af1deb95aaea6eb3b9adf9523790be52d7f41" From fbf4143674b70126dacd85c2598dc1c071d292bc Mon Sep 17 00:00:00 2001 From: "Jonathan C. McKinney" Date: Sat, 11 May 2024 01:32:01 -0700 Subject: [PATCH 7/9] Fix input and output tokens counting for summarization --- src/gpt_langchain.py | 8 +++++--- src/version.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/gpt_langchain.py b/src/gpt_langchain.py index e003aa9bd..6ddd66a5e 100644 --- a/src/gpt_langchain.py +++ b/src/gpt_langchain.py @@ -1703,6 +1703,8 @@ def reducer(accumulator, element): llm_output = {"token_usage": token_usage, "model_name": self.model_name} self.count_output_tokens += token_usage.get('completion_tokens', 0) + if self.count_output_tokens == 0: + self.count_output_tokens += sum([self.get_num_tokens(x[0].text) for x in generations if len(x) > 0]) return LLMResult(generations=generations, llm_output=llm_output) def _generate( @@ -8463,11 +8465,11 @@ def get_sources_answer(query, docs, answer, answer_sources) if verbose or True: if t_run is not None and int(t_run) > 0: - sorted_sources_urls += 'Total Time: %d [s]

' % t_run + sorted_sources_urls += 'Total Time: %d [s]
' % t_run if count_input_tokens and count_output_tokens: - sorted_sources_urls += 'Input Tokens: %s | Output Tokens: %d

' % ( + sorted_sources_urls += 'Input Tokens: %s | Output Tokens: %d
' % ( count_input_tokens, count_output_tokens) - sorted_sources_urls += "Total document chunks used: %s

" % len(docs) + sorted_sources_urls += "Total document chunks used: %s
" % len(docs) sorted_sources_urls += f"

{source_postfix}" title_overall = "Sources" sorted_sources_urls = f"""
{title_overall}{sorted_sources_urls}
""" diff --git a/src/version.py b/src/version.py index be9662c71..b7f9ced39 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1 @@ -__version__ = "8d2af1deb95aaea6eb3b9adf9523790be52d7f41" +__version__ = "ad975db77c5f6387a43207935dedb65d3db60fd6" From d86538bd2d460ff65ca1d4b4e83da9316bbcab10 Mon Sep 17 00:00:00 2001 From: "Jonathan C. McKinney" Date: Sat, 11 May 2024 01:40:32 -0700 Subject: [PATCH 8/9] Update test --- src/version.py | 2 +- tests/test_client_calls.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/version.py b/src/version.py index b7f9ced39..b848803b7 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1 @@ -__version__ = "ad975db77c5f6387a43207935dedb65d3db60fd6" +__version__ = "fbf4143674b70126dacd85c2598dc1c071d292bc" diff --git a/tests/test_client_calls.py b/tests/test_client_calls.py index 02a6822b5..ebfb2a841 100644 --- a/tests/test_client_calls.py +++ b/tests/test_client_calls.py @@ -2241,7 +2241,8 @@ def test_client_chat_stream_langchain_steps3(loaders, enforce_h2ogpt_api_key, en "more text is boring" in res_dict['response'] or "more text is boring" in res_dict['response'] or "it can be inferred that more text is indeed boring" in res_dict['response'] or - "expressing frustration" in res_dict['response']) \ + "expressing frustration" in res_dict['response'] or + "it seems that more text can indeed be boring" in res_dict['response']) \ and 'sample1.pdf' in res_dict['response'] # QUERY2 prompt = "What is a universal file format?" From 819ea415a1c5429c27c8337977588bb9543f5d69 Mon Sep 17 00:00:00 2001 From: "Jonathan C. McKinney" Date: Sat, 11 May 2024 01:46:26 -0700 Subject: [PATCH 9/9] Fix exception --- src/utils_langchain.py | 2 +- src/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils_langchain.py b/src/utils_langchain.py index f1089e6fb..c008973b8 100644 --- a/src/utils_langchain.py +++ b/src/utils_langchain.py @@ -317,7 +317,7 @@ def _chunk_sources(sources, chunk=True, chunk_size=512, language=None, db_type=N hf_embedding_model=hf_embedding_model if not use_openai_embedding else None, verbose=verbose) source_chunks = [x[0] for x in source_chunks_with_score] - except BaseException: + except BaseException as e: if os.getenv('HARD_ASSERTS'): raise print("Failed to split with new method, use old method: %s" % str(e)) diff --git a/src/version.py b/src/version.py index b848803b7..61549abe7 100644 --- a/src/version.py +++ b/src/version.py @@ -1 +1 @@ -__version__ = "fbf4143674b70126dacd85c2598dc1c071d292bc" +__version__ = "d86538bd2d460ff65ca1d4b4e83da9316bbcab10"